You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/08/04 14:40:57 UTC
[23/23] ambari git commit: Merge branch 'branch-2.6' into
branch-feature-AMBARI-21450
Merge branch 'branch-2.6' into branch-feature-AMBARI-21450
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/00712125
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/00712125
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/00712125
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 00712125d7ac3f188a4eb3174b9955551e202b9b
Parents: abd6459 7efb9de
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Aug 4 10:40:05 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Aug 4 10:40:05 2017 -0400
----------------------------------------------------------------------
.../libraries/functions/version.py | 6 +
.../ambari-metrics/datasource.js | 35 +-
.../MetricCollectorHAController.java | 42 +-
.../checks/DatabaseConsistencyCheckHelper.java | 115 ++++
.../AmbariManagementControllerImpl.java | 2 +-
.../ClusterStackVersionResourceProvider.java | 50 +-
.../controller/utilities/DatabaseChecker.java | 296 ----------
.../state/configgroup/ConfigGroupImpl.java | 6 +-
.../HDF/grafana-nifi-hosts.json | 17 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 2 +-
.../2.1.0.2.0/package/scripts/params_windows.py | 2 +-
.../HDFS/2.1.0.2.0/package/scripts/utils.py | 16 +
.../0.4.0/package/scripts/setup_ranger_xml.py | 6 +
.../src/main/resources/scripts/stack_advisor.py | 7 +
.../0.8/services/HDFS/package/scripts/params.py | 2 +-
.../4.0/properties/stack_features.json | 31 +-
.../package/alerts/alert_checkpoint_time.py | 255 +++++++++
.../alerts/alert_datanode_unmounted_data_dir.py | 177 ++++++
.../package/alerts/alert_ha_namenode_health.py | 243 ++++++++
.../package/alerts/alert_metrics_deviation.py | 470 ++++++++++++++++
.../package/alerts/alert_upgrade_finalized.py | 179 ++++++
.../services/HDFS/package/files/checkWebUI.py | 86 +++
.../services/HDFS/package/scripts/__init__.py | 20 +
.../scripts/balancer-emulator/hdfs-command.py | 45 ++
.../services/HDFS/package/scripts/datanode.py | 174 ++++++
.../HDFS/package/scripts/datanode_upgrade.py | 156 +++++
.../4.2.5/services/HDFS/package/scripts/hdfs.py | 173 ++++++
.../HDFS/package/scripts/hdfs_client.py | 123 ++++
.../HDFS/package/scripts/hdfs_datanode.py | 84 +++
.../HDFS/package/scripts/hdfs_namenode.py | 562 +++++++++++++++++++
.../HDFS/package/scripts/hdfs_nfsgateway.py | 75 +++
.../HDFS/package/scripts/hdfs_rebalance.py | 130 +++++
.../HDFS/package/scripts/hdfs_snamenode.py | 64 +++
.../HDFS/package/scripts/install_params.py | 38 ++
.../HDFS/package/scripts/journalnode.py | 198 +++++++
.../HDFS/package/scripts/journalnode_upgrade.py | 152 +++++
.../services/HDFS/package/scripts/namenode.py | 420 ++++++++++++++
.../HDFS/package/scripts/namenode_ha_state.py | 219 ++++++++
.../HDFS/package/scripts/namenode_upgrade.py | 322 +++++++++++
.../services/HDFS/package/scripts/nfsgateway.py | 147 +++++
.../services/HDFS/package/scripts/params.py | 30 +
.../HDFS/package/scripts/params_linux.py | 527 +++++++++++++++++
.../HDFS/package/scripts/params_windows.py | 76 +++
.../HDFS/package/scripts/service_check.py | 153 +++++
.../HDFS/package/scripts/setup_ranger_hdfs.py | 121 ++++
.../services/HDFS/package/scripts/snamenode.py | 152 +++++
.../HDFS/package/scripts/status_params.py | 58 ++
.../services/HDFS/package/scripts/utils.py | 385 +++++++++++++
.../services/HDFS/package/scripts/zkfc_slave.py | 200 +++++++
.../package/templates/exclude_hosts_list.j2 | 21 +
.../HDFS/package/templates/hdfs.conf.j2 | 35 ++
.../services/HDFS/package/templates/slaves.j2 | 21 +
.../4.2.5/upgrades/config-upgrade.xml | 59 ++
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 21 +
.../BigInsights/4.2/upgrades/config-upgrade.xml | 71 ++-
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 29 +-
.../DatabaseConsistencyCheckHelperTest.java | 77 +++
.../AmbariManagementControllerTest.java | 2 +-
...ClusterStackVersionResourceProviderTest.java | 11 +-
.../utilities/DatabaseCheckerTest.java | 105 ----
.../app/controllers/wizard/step1_controller.js | 2 +
.../app/mixins/common/configs/configs_saver.js | 12 +-
.../mixins/common/configs/enhanced_configs.js | 12 +-
ambari-web/app/routes/installer.js | 4 +-
ambari-web/app/templates/wizard/step1.hbs | 342 +++++------
.../app/views/main/alert_definitions_view.js | 5 +-
ambari-web/app/views/wizard/step1_view.js | 2 +-
.../test/controllers/wizard/step1_test.js | 2 +
.../mixins/common/configs/configs_saver_test.js | 19 +
.../common/configs/enhanced_configs_test.js | 10 +
.../views/main/alert_definitions_view_test.js | 68 +++
ambari-web/test/views/wizard/step1_view_test.js | 2 +-
72 files changed, 7156 insertions(+), 625 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index f07ecad,b489d7f..ed509b7
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@@ -706,4 -992,144 +707,47 @@@ public class ClusterStackVersionResourc
}
- /**
- * Ensures that the stack tools and stack features are set on
- * {@link ConfigHelper#CLUSTER_ENV} for the stack of the repository being
- * distributed. This step ensures that the new repository can be distributed
- * with the correct tools.
- * <p/>
- * If the cluster's current stack name matches that of the new stack or the
- * new stack's tools are already added in the configuration, then this method
- * will not change anything.
- *
- * @param stackId
- * the stack of the repository being distributed (not {@code null}).
- * @param cluster
- * the cluster the new stack/repo is being distributed for (not
- * {@code null}).
- * @throws AmbariException
- */
- private void bootstrapStackTools(StackId stackId, Cluster cluster) throws AmbariException {
- // if the stack name is the same as the cluster's current stack name, then
- // there's no work to do
- if (StringUtils.equals(stackId.getStackName(),
- cluster.getCurrentStackVersion().getStackName())) {
- return;
- }
-
- ConfigHelper configHelper = configHelperProvider.get();
-
- // get the stack tools/features for the stack being distributed
- Map<String, Map<String, String>> defaultStackConfigurationsByType = configHelper.getDefaultProperties(
- stackId, cluster);
-
- Map<String, String> clusterEnvDefaults = defaultStackConfigurationsByType.get(
- ConfigHelper.CLUSTER_ENV);
-
- Config clusterEnv = cluster.getDesiredConfigByType(ConfigHelper.CLUSTER_ENV);
- Map<String, String> clusterEnvProperties = clusterEnv.getProperties();
-
- // the 3 properties we need to check and update
- Set<String> properties = Sets.newHashSet(ConfigHelper.CLUSTER_ENV_STACK_ROOT_PROPERTY,
- ConfigHelper.CLUSTER_ENV_STACK_TOOLS_PROPERTY,
- ConfigHelper.CLUSTER_ENV_STACK_FEATURES_PROPERTY);
-
- // any updates are stored here and merged into the existing config type
- Map<String, String> updatedProperties = new HashMap<>();
-
- for (String property : properties) {
- // determine if the property exists in the stack being distributed (it
- // kind of has to, but we'll be safe if it's not found)
- String newStackDefaultJson = clusterEnvDefaults.get(property);
- if (StringUtils.isBlank(newStackDefaultJson)) {
- continue;
- }
-
- String existingPropertyJson = clusterEnvProperties.get(property);
-
- // if the stack tools/features property doesn't exist, then just set the
- // one from the new stack
- if (StringUtils.isBlank(existingPropertyJson)) {
- updatedProperties.put(property, newStackDefaultJson);
- continue;
- }
-
- // now is the hard part - we need to check to see if the new stack tools
- // exists alongside the current tools and if it doesn't, then add the new
- // tools in
- final Map<String, Object> existingJson;
- final Map<String, ?> newStackJsonAsObject;
- if (StringUtils.equals(property, ConfigHelper.CLUSTER_ENV_STACK_ROOT_PROPERTY)) {
- existingJson = gson.<Map<String, Object>> fromJson(existingPropertyJson, Map.class);
- newStackJsonAsObject = gson.<Map<String, String>> fromJson(newStackDefaultJson, Map.class);
- } else {
- existingJson = gson.<Map<String, Object>> fromJson(existingPropertyJson,
- Map.class);
-
- newStackJsonAsObject = gson.<Map<String, Map<Object, Object>>> fromJson(newStackDefaultJson,
- Map.class);
- }
-
- if (existingJson.keySet().contains(stackId.getStackName())) {
- continue;
- }
-
- existingJson.put(stackId.getStackName(), newStackJsonAsObject.get(stackId.getStackName()));
-
- String newJson = gson.toJson(existingJson);
- updatedProperties.put(property, newJson);
- }
-
- if (!updatedProperties.isEmpty()) {
- AmbariManagementController amc = getManagementController();
- String serviceNote = String.format(
- "Adding stack tools for %s while distributing a new repository", stackId.toString());
-
- configHelper.updateConfigType(cluster, amc, clusterEnv.getType(), updatedProperties, null,
- amc.getAuthName(), serviceNote);
- }
- }
+
+ /**
+ * Check one host is enough to tell the arch
+ * because all hosts should have the same arch.
+ * @param hosts List<Host>
+ * @return osFamily, null if hosts is empty or is X86_64
+
+ */
+ private String getPowerPCOsFamily(List<Host> hosts) {
+ if (hosts.isEmpty()){
+ return null;
+ } else {
+ Host host = hosts.get(0);
+ String osFamily = host.getHostAttributes().get("os_family");
+ if (null != osFamily && osFamily.endsWith("-ppc")){
+ return osFamily;
+ } else {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Use os type with -ppc post fix for powerpc
+ * in order to have it consistent with the os information
+ * stored in the Hosts table
+ * No need to apply the change if os is x86_64
+ * */
+ private String getOsTypeForRepo(OperatingSystemEntity operatingSystem, String osFamily) {
+ if (null != osFamily){
+ String osType = operatingSystem.getOsType();
+ int pos = osFamily.indexOf("-ppc");
+ if (pos > 0){
+ String os = osType.substring(0, pos);
+ String majorVersion = osType.substring(os.length());
+ return String.format("%s-ppc%s", os, majorVersion);
+ } else {
+ return operatingSystem.getOsType();
+ }
+ } else {
+ return operatingSystem.getOsType();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00712125/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
index 32a5358,68596e0..41c83e4
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
@@@ -226,13 -233,9 +226,13 @@@ public class ClusterStackVersionResourc
MaintenanceState.OFF).anyTimes();
expect(host.getAllHostVersions()).andReturn(
Collections.<HostVersionEntity>emptyList()).anyTimes();
-
+ expect(host.getHostAttributes()).andReturn(new HashMap<String, String>()).anyTimes();
replay(host);
hostsForCluster.put(hostname, host);
+
+ if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) {
+ hostsNeedingInstallCommands.add(host);
+ }
}
final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class);
@@@ -641,14 -657,9 +641,14 @@@
MaintenanceState.OFF).anyTimes();
expect(host.getAllHostVersions()).andReturn(
Collections.<HostVersionEntity>emptyList()).anyTimes();
-
+ expect(host.getHostAttributes()).andReturn(new HashMap<String, String>()).anyTimes();
replay(host);
hostsForCluster.put(hostname, host);
+
+
+ if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) {
+ hostsNeedingInstallCommands.add(host);
+ }
}
Service hdfsService = createNiceMock(Service.class);
@@@ -882,13 -888,9 +882,13 @@@
MaintenanceState.OFF).anyTimes();
expect(host.getAllHostVersions()).andReturn(
Collections.<HostVersionEntity>emptyList()).anyTimes();
-
+ expect(host.getHostAttributes()).andReturn(new HashMap<String, String>()).anyTimes();
replay(host);
hostsForCluster.put(hostname, host);
+
+ if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) {
+ hostsNeedingInstallCommands.add(host);
+ }
}
Service hdfsService = createNiceMock(Service.class);
@@@ -1130,9 -1433,10 +1130,9 @@@
expect(host.getOsFamily()).andReturn("redhat6").anyTimes();
expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn(
MaintenanceState.OFF).anyTimes();
- expect(host.getAllHostVersions()).andReturn(
- Collections.<HostVersionEntity>emptyList()).anyTimes();
+ expect(host.getAllHostVersions()).andReturn(hostVersions).anyTimes();
-
+ expect(host.getHostAttributes()).andReturn(new HashMap<String, String>()).anyTimes();
- replay(host);
+ replay(host, hostVersion);
hostsForCluster.put(hostname, host);
}