You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/06/01 16:26:23 UTC
ambari git commit: AMBARI-16971. Stack VDF should ignore unsupported
OS (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 7033b1c54 -> 99a615566
AMBARI-16971. Stack VDF should ignore unsupported OS (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/99a61556
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/99a61556
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/99a61556
Branch: refs/heads/trunk
Commit: 99a615566dc36c1b54e6c13a268149e49a952b50
Parents: 7033b1c
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue May 31 15:44:40 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Wed Jun 1 11:42:56 2016 -0400
----------------------------------------------------------------------
.../ambari_commons/resources/os_family.json | 3 +-
.../server/state/stack/LatestRepoCallable.java | 28 +++++++++++
.../ambari/server/state/stack/OsFamily.java | 44 +++++++++++------
.../VersionDefinitionResourceProviderTest.java | 17 +++++++
.../resources/stacks/HDP/2.2.0/repos/hdp.json | 4 +-
.../2.2.0/repos/version-2.2.0.4-124-suse11.xml | 51 ++++++++++++++++++++
6 files changed, 129 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-common/src/main/python/ambari_commons/resources/os_family.json
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/resources/os_family.json b/ambari-common/src/main/python/ambari_commons/resources/os_family.json
index c7b38aa..e9bd1a4 100644
--- a/ambari-common/src/main/python/ambari_commons/resources/os_family.json
+++ b/ambari-common/src/main/python/ambari_commons/resources/os_family.json
@@ -67,6 +67,7 @@
},
"aliases": {
"amazon2015": "amazon6",
- "amazon2016": "amazon6"
+ "amazon2016": "amazon6",
+ "suse11sp3": "suse11"
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java
index a332308..3c7c001 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/LatestRepoCallable.java
@@ -22,6 +22,7 @@ import java.io.FileReader;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.net.URI;
+import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -32,6 +33,7 @@ import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -190,9 +192,35 @@ public class LatestRepoCallable implements Callable<Void> {
* @throws Exception
*/
private VersionDefinitionXml mergeDefinitions(StackId stackId, String version, Map<String, String> osMap) throws Exception {
+
+ Set<String> oses = new HashSet<>();
+ for (RepositoryInfo ri : stack.getRepositories()) {
+ if (null != os_family.find(ri.getOsType())) {
+ oses.add(os_family.find(ri.getOsType()));
+ }
+ }
+
VersionDefinitionXml.Merger merger = new VersionDefinitionXml.Merger();
for (Entry<String, String> versionEntry : osMap.entrySet()) {
+
+ String osFamily = os_family.find(versionEntry.getKey());
+
+ // !!! check for aliases. Moving this to OsFamily could result in incorrect behavior
+ if (null == osFamily) {
+ String alias = os_family.getAliases().get(versionEntry.getKey());
+ if (null != alias) {
+ osFamily = os_family.find(alias);
+ }
+ }
+
+ // !!! if the family is not known OR not part of the stack, skip
+ if (null == osFamily || !oses.contains(osFamily)) {
+ LOG.info("Stack {} cannot resolve OS {} to the supported ones: {}. Family: {}",
+ stackId, versionEntry.getKey(), StringUtils.join(oses, ','), osFamily);
+ continue;
+ }
+
String uriString = versionEntry.getValue();
if ('.' == uriString.charAt(0)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
index e494c44..599696a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
@@ -21,20 +21,23 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.inject.Singleton;
-
+import org.apache.ambari.server.configuration.Configuration;
+import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
-
-import org.apache.ambari.server.configuration.Configuration;
-import org.apache.commons.io.IOUtils;
+import com.google.inject.Singleton;
/**
* Class that encapsulates OS family logic
@@ -44,12 +47,10 @@ public class OsFamily {
private final static String OS_FAMILY_UBUNTU = "ubuntu";
private final static String OS_FAMILY_SUSE = "suse";
private final static String OS_FAMILY_REDHAT = "redhat";
-
+
private final String os_pattern = "([\\D]+|(?:[\\D]+[\\d]+[\\D]+))([\\d]*)";
private final String OS_DISTRO = "distro";
private final String OS_VERSION = "versions";
- private final String OS_MAPPING = "mapping";
- private final String OS_ALIASES = "aliases";
private final String LOAD_CONFIG_MSG = "Could not load OS family definition from %s file";
private final String FILE_NAME = "os_family.json";
private final Logger LOG = LoggerFactory.getLogger(OsFamily.class);
@@ -77,7 +78,9 @@ public class OsFamily {
FileInputStream inputStream = null;
try {
File f = new File(SharedResourcesPath, FILE_NAME);
- if (!f.exists()) throw new Exception();
+ if (!f.exists()) {
+ throw new Exception();
+ }
inputStream = new FileInputStream(f);
Type type = new TypeToken<JsonOsFamilyRoot>() {}.getType();
@@ -103,7 +106,7 @@ public class OsFamily {
Pattern r = Pattern.compile(os_pattern);
Matcher m = r.matcher(os);
- if (m.matches()){
+ if (m.matches()) {
pos.put(OS_DISTRO, m.group(1));
pos.put(OS_VERSION, m.group(2));
} else {
@@ -144,9 +147,10 @@ public class OsFamily {
return family + pos.get(OS_VERSION);
}
}
+
return null;
}
-
+
/**
* Finds the family for the specific OS
* @param os the OS
@@ -178,15 +182,15 @@ public class OsFamily {
}
return r;
}
-
+
public boolean isUbuntuFamily(String osType) {
return isOsInFamily(osType, OS_FAMILY_UBUNTU);
}
-
+
public boolean isSuseFamily(String osType) {
return isOsInFamily(osType, OS_FAMILY_SUSE);
}
-
+
public boolean isRedhatFamily(String osType) {
return isOsInFamily(osType, OS_FAMILY_REDHAT);
}
@@ -199,8 +203,16 @@ public class OsFamily {
private boolean isFamilyExtendedByFamily(String currentFamily, String family) {
return (currentFamily.equals(family) || getOsFamilyParent(currentFamily)!=null && isFamilyExtendedByFamily(getOsFamilyParent(currentFamily), family));
}
-
+
private String getOsFamilyParent(String osFamily) {
return osMap.get(osFamily).getExtendsFamily();
}
+
+ /**
+ * @return the map of aliases
+ */
+ public Map<String, String> getAliases() {
+ return (null == jsonOsFamily || null == jsonOsFamily.getAliases()) ?
+ Collections.<String, String>emptyMap() : jsonOsFamily.getAliases();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java
index 9f7a90f..75427ff 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java
@@ -46,6 +46,7 @@ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.security.TestAuthenticationFactory;
import org.apache.ambari.server.stack.StackManager;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.junit.After;
@@ -55,6 +56,7 @@ import org.junit.Test;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
+import com.google.common.collect.Sets;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.persist.PersistService;
@@ -279,6 +281,21 @@ public class VersionDefinitionResourceProviderTest {
found1 = true;
} else if ("HDP-2.2.0".equals(res.getPropertyValue("VersionDefinition/id"))) {
Assert.assertEquals(Boolean.TRUE, res.getPropertyValue("VersionDefinition/stack_default"));
+
+ VersionDefinitionXml vdf = ami.getVersionDefinition("HDP-2.2.0");
+
+ Assert.assertNotNull(vdf);
+ Assert.assertEquals(2, vdf.repositoryInfo.getOses().size());
+
+ String family1 = vdf.repositoryInfo.getOses().get(0).getFamily();
+ String family2 = vdf.repositoryInfo.getOses().get(1).getFamily();
+
+ Assert.assertFalse(family1.equals(family2));
+ Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family1));
+ Assert.assertTrue(Sets.newHashSet("suse11", "redhat6").contains(family2));
+
+
+
found2 = true;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json
index f80fc0a..708d757 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/hdp.json
@@ -8,7 +8,9 @@
},
"manifests": {
"2.2.1.0": {
- "centos6": "./version-2.2.0.4-123.xml"
+ "centos6": "./version-2.2.0.4-123.xml",
+ "debian6": "./version-2.2.0.4-123.xml",
+ "suse11sp3": "./version-2.2.0.4-123-suse11.xml"
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/99a61556/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.4-124-suse11.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.4-124-suse11.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.4-124-suse11.xml
new file mode 100644
index 0000000..68d811a
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/repos/version-2.2.0.4-124-suse11.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="version_definition.xsd">
+ <release>
+ <type>STANDARD</type>
+ <stack-id>HDP-2.2.0</stack-id>
+ <version>2.2.0.4</version>
+ <build>123</build>
+ <release-notes>http://example.com</release-notes>
+ <display>HDP-2.2.0.4-1234</display>
+ <compatible-with>2.2.0.0</compatible-with>
+ </release>
+ <manifest>
+ <service id="HDFS-271" name="HDFS" version="2.7.1.2.4"/>
+ <service id="HBASE-132" name="HBASE" version="1.3.2.4.3"/>
+ </manifest>
+ <available-services>
+ <service idref="HDFS-271"/>
+ </available-services>
+ <repository-info>
+ <os family="suse11">
+ <repo>
+ <baseurl>http://baseurl3</baseurl>
+ <repoid>HDP-2.4</repoid>
+ <reponame>HDP</reponame>
+ </repo>
+ <repo>
+ <baseurl>http://baseurl4</baseurl>
+ <repoid>HDP-UTILS-1.1.0.20</repoid>
+ <reponame>HDP-UTILS</reponame>
+ </repo>
+ </os>
+
+ </repository-info>
+
+</repository-version>