You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/10/27 15:20:29 UTC
[01/16] ambari git commit: AMBARI-18651. HDP-2.5 installation allows
ZKFC to advertise version (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-18634 ba526d9a2 -> 6088e4f1c
AMBARI-18651. HDP-2.5 installation allows ZKFC to advertise version (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a5b8230a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a5b8230a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a5b8230a
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: a5b8230aef0f12a52d1bbcd1cb5cf72900476d6c
Parents: 70d8da7
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Oct 26 18:53:16 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Wed Oct 26 18:53:50 2016 +0300
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../ambari/server/events/AmbariEvent.java | 5 +
.../server/events/StackUpgradeFinishEvent.java | 41 +++++++
.../upgrade/StackUpgradeFinishListener.java | 83 ++++++++++++++
.../listeners/upgrade/StackVersionListener.java | 53 ++++++---
.../publishers/VersionEventPublisher.java | 9 +-
.../upgrades/FinalizeUpgradeAction.java | 8 ++
.../ambari/server/state/ServiceComponent.java | 6 ++
.../server/state/ServiceComponentImpl.java | 41 +++----
.../2.1.0.2.0/package/scripts/zkfc_slave.py | 20 ++++
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../stacks/HDP/2.5/services/HDFS/metainfo.xml | 6 ++
.../upgrade/StackUpgradeFinishListenerTest.java | 108 +++++++++++++++++++
.../upgrade/StackVersionListenerTest.java | 47 ++++++--
14 files changed, 384 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 1396bd8..d1428d4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -101,3 +101,4 @@ class StackFeature:
SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support"
ATLAS_HBASE_SETUP = "atlas_hbase_setup"
RANGER_HIVE_PLUGIN_JDBC_URL = "ranger_hive_plugin_jdbc_url"
+ ZKFC_VERSION_ADVERTISED = "zkfc_version_advertised"
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java
index de9e7b6..0ff7e8a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/AmbariEvent.java
@@ -118,6 +118,11 @@ public abstract class AmbariEvent {
SERVICE_COMPONENT_RECOVERY_CHANGED,
/**
+ * Stack upgrade or downgrade finishes
+ */
+ FINALIZE_UPGRADE_FINISH,
+
+ /**
* Cluster configuration changed.
*/
CLUSTER_CONFIG_CHANGED,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java
new file mode 100644
index 0000000..d5745dc
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/StackUpgradeFinishEvent.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.events;
+
+import org.apache.ambari.server.state.Cluster;
+
+public class StackUpgradeFinishEvent extends ClusterEvent {
+
+
+ public Cluster getCluster() {
+ return cluster;
+ }
+
+ protected final Cluster cluster;
+
+ /**
+ * Constructor.
+ *
+ * @param cluster
+ */
+ public StackUpgradeFinishEvent(Cluster cluster) {
+ super(AmbariEventType.FINALIZE_UPGRADE_FINISH, cluster.getClusterId());
+ this.cluster = cluster;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java
new file mode 100644
index 0000000..b1bffef
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListener.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.events.listeners.upgrade;
+
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.events.StackUpgradeFinishEvent;
+import org.apache.ambari.server.events.publishers.VersionEventPublisher;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.eventbus.AllowConcurrentEvents;
+import com.google.common.eventbus.Subscribe;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+/**
+ * The {@link StackUpgradeFinishListener} class handles updating component info
+ * after stack upgrade or downgrade finish
+ */
+@Singleton
+@EagerSingleton
+public class StackUpgradeFinishListener {
+ /**
+ * Logger.
+ */
+ private final static Logger LOG = LoggerFactory.getLogger(StackUpgradeFinishListener.class);
+ @Inject
+ Provider<AmbariMetaInfo> ambariMetaInfo;
+
+ /**
+ * Constructor.
+ *
+ * @param eventPublisher the publisher
+ */
+ @Inject
+ public StackUpgradeFinishListener(VersionEventPublisher eventPublisher) {
+ eventPublisher.register(this);
+ }
+
+ @Subscribe
+ @AllowConcurrentEvents
+ public void onAmbariEvent(StackUpgradeFinishEvent event) {
+ LOG.debug("Received event {}", event);
+
+ Cluster cluster = event.getCluster();
+
+ //update component info due to new stack
+ for (Service service : cluster.getServices().values()) {
+ for (ServiceComponent sc : service.getServiceComponents().values()) {
+ try {
+ sc.updateComponentInfo();
+ } catch (AmbariException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Caught AmbariException when update component info", e);
+ }
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
index 87247eb..f5a5b0c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListener.java
@@ -22,11 +22,13 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.EagerSingleton;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.State;
@@ -38,6 +40,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import com.google.inject.Singleton;
/**
@@ -66,6 +69,9 @@ public class StackVersionListener {
@Inject
private RepositoryVersionDAO repositoryVersionDAO;
+ @Inject
+ Provider<AmbariMetaInfo> ambariMetaInfo;
+
/**
* Constructor.
*
@@ -110,8 +116,14 @@ public class StackVersionListener {
// Update host component version value if needed
try {
+ AmbariMetaInfo metaInfo = ambariMetaInfo.get();
+ ComponentInfo componentInfo = metaInfo.getComponent(cluster.getDesiredStackVersion().getStackName(),
+ cluster.getDesiredStackVersion().getStackVersion(), sch.getServiceName(), sch.getServiceComponentName());
ServiceComponent sc = cluster.getService(sch.getServiceName()).getServiceComponent(sch.getServiceComponentName());
- if(!sc.isVersionAdvertised() && StringUtils.isNotBlank(newVersion)
+ if (componentInfo.isVersionAdvertised() && StringUtils.isNotBlank(newVersion)
+ && !UNKNOWN_VERSION.equalsIgnoreCase(newVersion)) {
+ processComponentAdvertisedVersion(cluster, sch, newVersion, sc);
+ } else if(!sc.isVersionAdvertised() && StringUtils.isNotBlank(newVersion)
&& !UNKNOWN_VERSION.equalsIgnoreCase(newVersion)) {
LOG.error("ServiceComponent {0} doesn't advertise version, " +
"however ServiceHostComponent {} on host {} advertised version as {}. Skipping version update",
@@ -119,17 +131,8 @@ public class StackVersionListener {
} else {
if (UNKNOWN_VERSION.equals(sc.getDesiredVersion())) {
processUnknownDesiredVersion(cluster, sc, sch, newVersion);
- } else if (StringUtils.isNotBlank(newVersion)) {
- String previousVersion = sch.getVersion();
- if (previousVersion == null || UNKNOWN_VERSION.equalsIgnoreCase(previousVersion)) {
- // value may be "UNKNOWN" when upgrading from older Ambari versions
- // or if host component reports it's version for the first time
- sch.setUpgradeState(UpgradeState.NONE);
- sch.setVersion(newVersion);
- bootstrapVersion(cluster, sch);
- } else if (!StringUtils.equals(previousVersion, newVersion)) { //
- processComponentVersionChange(cluster, sc, sch, newVersion);
- }
+ } else {
+ processComponentAdvertisedVersion(cluster, sch, newVersion, sc);
}
}
} catch (Exception e) {
@@ -142,6 +145,32 @@ public class StackVersionListener {
}
/**
+ * Update host component version
+ * or
+ * Bootstrap cluster/repo version when version is reported for the first time
+ * @param cluster target cluster
+ * @param sch target host component
+ * @param newVersion advertised version
+ * @param sc target service component
+ * @throws AmbariException
+ */
+ private void processComponentAdvertisedVersion(Cluster cluster, ServiceComponentHost sch, String newVersion, ServiceComponent sc) throws AmbariException {
+ if (StringUtils.isBlank(newVersion)) {
+ return;
+ }
+ String previousVersion = sch.getVersion();
+ if (previousVersion == null || UNKNOWN_VERSION.equalsIgnoreCase(previousVersion)) {
+ // value may be "UNKNOWN" when upgrading from older Ambari versions
+ // or if host component reports it's version for the first time
+ sch.setUpgradeState(UpgradeState.NONE);
+ sch.setVersion(newVersion);
+ bootstrapVersion(cluster, sch);
+ } else if (!StringUtils.equals(previousVersion, newVersion)) {
+ processComponentVersionChange(cluster, sc, sch, newVersion);
+ }
+ }
+
+ /**
* Bootstrap cluster/repo version when version is reported for the first time
* @param cluster target cluster
* @param sch target host component
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java
index 5b32c4e..710707e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/VersionEventPublisher.java
@@ -20,11 +20,12 @@ package org.apache.ambari.server.events.publishers;
import com.google.common.eventbus.EventBus;
import com.google.inject.Singleton;
-import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
+
+import org.apache.ambari.server.events.ClusterEvent;
/**
* The {@link VersionEventPublisher} is used to publish instances of
- * {@link HostComponentVersionAdvertisedEvent} to any {@link com.google.common.eventbus.Subscribe} interested.
+ * {@link ClusterEvent} to any {@link com.google.common.eventbus.Subscribe} interested.
* It uses a single-threaded, serial {@link EventBus}.
*/
@Singleton
@@ -44,11 +45,11 @@ public class VersionEventPublisher {
/**
* Publishes the specified event to all registered listeners that
* {@link com.google.common.eventbus.Subscribe} to any of the
- * {@link HostComponentVersionAdvertisedEvent} instances.
+ * {@link ClusterEvent} instances.
*
* @param event the event
*/
- public void publish(HostComponentVersionAdvertisedEvent event) {
+ public void publish(ClusterEvent event) {
m_eventBus.post(event);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
index a07d0e6..e73651e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java
@@ -32,6 +32,9 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
+import org.apache.ambari.server.events.StackUpgradeFinishEvent;
+import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
import org.apache.ambari.server.orm.dao.HostComponentStateDAO;
import org.apache.ambari.server.orm.dao.HostVersionDAO;
@@ -126,6 +129,9 @@ public class FinalizeUpgradeAction extends AbstractServerAction {
@Inject
private AmbariMetaInfo ambariMetaInfo;
+ @Inject
+ VersionEventPublisher versionEventPublisher;
+
@Override
public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
throws AmbariException, InterruptedException {
@@ -300,6 +306,7 @@ public class FinalizeUpgradeAction extends AbstractServerAction {
String.format("Finalizing the version for %d host(s).\n", hostVersionsAllowed.size()));
cluster.mapHostVersions(hostsToUpdate, upgradingClusterVersion, RepositoryVersionState.CURRENT);
+ versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster));
// Reset upgrade state
cluster.setUpgradeEntity(null);
@@ -447,6 +454,7 @@ public class FinalizeUpgradeAction extends AbstractServerAction {
// ensure that when downgrading, we set the desired back to the
// original value
cluster.setDesiredStackVersion(currentClusterStackId);
+ versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster));
// Reset upgrade state
cluster.setUpgradeEntity(null);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
index f91a958..e93ab9a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
@@ -59,6 +59,12 @@ public interface ServiceComponent {
void setDesiredVersion(String version);
+ /**
+ * Refresh Component info due to current stack
+ * @throws AmbariException
+ */
+ void updateComponentInfo() throws AmbariException;
+
Map<String, ServiceComponentHost> getServiceComponentHosts();
ServiceComponentHost getServiceComponentHost(String hostname)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
index f383e80..f9c0eb7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
@@ -60,10 +60,10 @@ public class ServiceComponentImpl implements ServiceComponent {
private final Service service;
private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
private final String componentName;
- private final String displayName;
- private final boolean isClientComponent;
- private final boolean isMasterComponent;
- private final boolean isVersionAdvertised;
+ private String displayName;
+ private boolean isClientComponent;
+ private boolean isMasterComponent;
+ private boolean isVersionAdvertised;
private final ServiceComponentDesiredStateDAO serviceComponentDesiredStateDAO;
@@ -73,6 +73,8 @@ public class ServiceComponentImpl implements ServiceComponent {
private final AmbariEventPublisher eventPublisher;
+ private AmbariMetaInfo ambariMetaInfo;
+
private final ConcurrentMap<String, ServiceComponentHost> hostComponents = new ConcurrentHashMap<String, ServiceComponentHost>();
/**
@@ -93,6 +95,7 @@ public class ServiceComponentImpl implements ServiceComponent {
StackDAO stackDAO, AmbariEventPublisher eventPublisher)
throws AmbariException {
+ this.ambariMetaInfo = ambariMetaInfo;
this.service = service;
this.componentName = componentName;
this.serviceComponentDesiredStateDAO = serviceComponentDesiredStateDAO;
@@ -113,6 +116,14 @@ public class ServiceComponentImpl implements ServiceComponent {
desiredStateEntity.setRecoveryEnabled(false);
desiredStateEntity.setDesiredStack(stackEntity);
+ updateComponentInfo();
+
+ persistEntities(desiredStateEntity);
+ desiredStateEntityId = desiredStateEntity.getId();
+ }
+
+ public void updateComponentInfo() throws AmbariException {
+ StackId stackId = service.getDesiredStackVersion();
try {
ComponentInfo compInfo = ambariMetaInfo.getComponent(stackId.getStackName(),
stackId.getStackVersion(), service.getName(), componentName);
@@ -128,9 +139,6 @@ public class ServiceComponentImpl implements ServiceComponent {
+ ", componentName=" + componentName
+ ", stackInfo=" + stackId.getStackId());
}
-
- persistEntities(desiredStateEntity);
- desiredStateEntityId = desiredStateEntity.getId();
}
@AssistedInject
@@ -149,27 +157,12 @@ public class ServiceComponentImpl implements ServiceComponent {
this.serviceComponentHostFactory = serviceComponentHostFactory;
this.stackDAO = stackDAO;
this.eventPublisher = eventPublisher;
+ this.ambariMetaInfo = ambariMetaInfo;
desiredStateEntityId = serviceComponentDesiredStateEntity.getId();
componentName = serviceComponentDesiredStateEntity.getComponentName();
- StackId stackId = service.getDesiredStackVersion();
- try {
- ComponentInfo compInfo = ambariMetaInfo.getComponent(
- stackId.getStackName(), stackId.getStackVersion(), service.getName(),
- componentName);
- isClientComponent = compInfo.isClient();
- isMasterComponent = compInfo.isMaster();
- isVersionAdvertised = compInfo.isVersionAdvertised();
- displayName = compInfo.getDisplayName();
- } catch (ObjectNotFoundException e) {
- throw new AmbariException("Trying to create a ServiceComponent"
- + " not recognized in stack info"
- + ", clusterName=" + service.getCluster().getClusterName()
- + ", serviceName=" + service.getName()
- + ", componentName=" + componentName
- + ", stackInfo=" + stackId.getStackId());
- }
+ updateComponentInfo();
for (HostComponentStateEntity hostComponentStateEntity : serviceComponentDesiredStateEntity.getHostComponentStateEntities()) {
HostComponentDesiredStateEntityPK pk = new HostComponentDesiredStateEntityPK();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
index b575d14..6b5e72e 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
@@ -27,15 +27,26 @@ from resource_management.core.exceptions import Fail
from resource_management.core.resources.system import Directory
from resource_management.core.resources.service import Service
from resource_management.core import shell
+from resource_management.libraries.functions import conf_select, stack_select
+from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.functions.security_commons import build_expectations
from resource_management.libraries.functions.security_commons import cached_kinit_executor
from resource_management.libraries.functions.security_commons import get_params_from_filesystem
from resource_management.libraries.functions.security_commons import validate_security_config_properties
from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.script import Script
+from resource_management.libraries.functions.version_select_util import get_component_version
class ZkfcSlave(Script):
+ def get_component_name(self):
+ import params
+ if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version):
+ return "hadoop-hdfs-zkfc"
+ pass
+
def install(self, env):
import params
env.set_params(params)
@@ -182,6 +193,15 @@ def initialize_ha_zookeeper(params):
Logger.error('HA state initialization in ZooKeeper threw an exception. Reason %s' %(str(ex)))
return False
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ Logger.info("Executing Stack Upgrade pre-restart")
+ import params
+ env.set_params(params)
+ if params.version and check_stack_feature(StackFeature.ZKFC_VERSION_ADVERTISED, params.version) \
+ and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+ conf_select.select(params.stack_name, "hadoop", params.version)
+ stack_select.select("hadoop-hdfs-zkfc", params.version)
+
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class ZkfcSlaveWindows(ZkfcSlave):
def start(self, env):
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index dbde58b..93e7bdf 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -308,6 +308,11 @@
"name": "ranger_hive_plugin_jdbc_url",
"description": "Handle Ranger hive repo config jdbc url change for stack 2.5 (AMBARI-18386)",
"min_version": "2.5.0.0"
+ },
+ {
+ "name": "zkfc_version_advertised",
+ "description": "ZKFC advertise version",
+ "min_version": "2.5.0.0"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml
index a3e4a64..89c128c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/metainfo.xml
@@ -21,6 +21,12 @@
<service>
<name>HDFS</name>
<version>2.7.1.2.5</version>
+ <components>
+ <component>
+ <name>ZKFC</name>
+ <versionAdvertised>true</versionAdvertised>
+ </component>
+ </components>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java
new file mode 100644
index 0000000..2bfd176
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.events.listeners.upgrade;
+
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.events.StackUpgradeFinishEvent;
+import org.apache.ambari.server.events.publishers.VersionEventPublisher;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+import org.apache.ambari.server.orm.entities.UpgradeEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeState;
+import org.easymock.EasyMockRunner;
+import org.easymock.EasyMockSupport;
+import org.easymock.TestSubject;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+
+
+/**
+ * StackVersionListener tests.
+ */
+@RunWith(EasyMockRunner.class)
+public class StackUpgradeFinishListenerTest extends EasyMockSupport {
+
+ private static final String INVALID_NEW_VERSION = "1.2.3.4-5678";
+ private static final String VALID_NEW_VERSION = "2.4.0.0-1000";
+ private static final String SERVICE_COMPONENT_NAME = "Some component name";
+ private static final String SERVICE_NAME = "Service name";
+ private static final Long CLUSTER_ID = 1L;
+ private static final String UNKNOWN_VERSION = "UNKNOWN";
+ private static final String VALID_PREVIOUS_VERSION = "2.2.0.0";
+ private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity();
+ private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity();
+ public static final String STACK_NAME = "HDP-2.4.0.0";
+ public static final String STACK_VERSION = "2.4.0.0";
+
+ private Cluster cluster;
+ private ServiceComponentHost sch;
+ private Service service;
+ private ServiceComponent serviceComponent;
+ private VersionEventPublisher publisher = new VersionEventPublisher();
+
+ @TestSubject
+ private StackUpgradeFinishListener listener = new StackUpgradeFinishListener(publisher);
+
+
+ @Before
+ public void setup() throws Exception {
+ cluster = createNiceMock(Cluster.class);
+ serviceComponent = createNiceMock(ServiceComponent.class);
+ service = createNiceMock(Service.class);
+ Map<String, Service> services = new HashMap<>();
+ services.put("mock_service",service);
+ Map<String, ServiceComponent> components = new HashMap<>();
+ components.put("mock_component", serviceComponent);
+
+ expect(cluster.getServices()).andReturn(services);
+ expect(service.getServiceComponents()).andReturn(components);
+ serviceComponent.updateComponentInfo();
+ }
+
+ @Test
+ public void testupdateComponentInfo() throws AmbariException {
+ replayAll();
+
+ sendEventAndVerify();
+ }
+
+
+ private void sendEventAndVerify() {
+ StackUpgradeFinishEvent event = new StackUpgradeFinishEvent(cluster);
+ listener.onAmbariEvent(event);
+
+ verifyAll();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5b8230a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
index d22622e..bd9a340 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackVersionListenerTest.java
@@ -17,31 +17,42 @@
*/
package org.apache.ambari.server.events.listeners.upgrade;
+import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import java.lang.reflect.Field;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.events.HostComponentVersionAdvertisedEvent;
import org.apache.ambari.server.events.publishers.VersionEventPublisher;
import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.UpgradeState;
+import org.easymock.EasyMockRunner;
import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.easymock.TestSubject;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.google.inject.Provider;
import junit.framework.Assert;
/**
* StackVersionListener tests.
*/
+@RunWith(EasyMockRunner.class)
public class StackVersionListenerTest extends EasyMockSupport {
private static final String INVALID_NEW_VERSION = "1.2.3.4-5678";
@@ -53,12 +64,23 @@ public class StackVersionListenerTest extends EasyMockSupport {
private static final String VALID_PREVIOUS_VERSION = "2.2.0.0";
private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity();
private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity();
+ public static final String STACK_NAME = "HDP-2.4.0.0";
+ public static final String STACK_VERSION = "2.4.0.0";
private Cluster cluster;
private ServiceComponentHost sch;
private Service service;
private ServiceComponent serviceComponent;
- private VersionEventPublisher publisher;
+ private VersionEventPublisher publisher = new VersionEventPublisher();
+ private AmbariMetaInfo ambariMetaInfo;
+ private ComponentInfo componentInfo;
+ private StackId stackId;
+
+ @TestSubject
+ private StackVersionListener listener = new StackVersionListener(publisher);
+
+ @Mock
+ private Provider<AmbariMetaInfo> ambariMetaInfoProvider;
@Before
public void setup() throws Exception {
@@ -66,13 +88,23 @@ public class StackVersionListenerTest extends EasyMockSupport {
sch = createNiceMock(ServiceComponentHost.class);
service = createNiceMock(Service.class);
serviceComponent = createNiceMock(ServiceComponent.class);
- publisher = createNiceMock(VersionEventPublisher.class);
+ componentInfo = createNiceMock(ComponentInfo.class);
+ stackId = createNiceMock(StackId.class);
+ ambariMetaInfo = createNiceMock(AmbariMetaInfo.class);
+
+ expect(ambariMetaInfoProvider.get()).andReturn(ambariMetaInfo);
+ expect(ambariMetaInfo.getComponent(anyString(),anyString(),anyString(),anyString())).andReturn(componentInfo);
+
+ expect(cluster.getDesiredStackVersion()).andReturn(stackId).atLeastOnce();
+ expect(stackId.getStackName()).andReturn(STACK_NAME);
+ expect(stackId.getStackVersion()).andReturn(STACK_VERSION);
expect(cluster.getClusterId()).andReturn(CLUSTER_ID);
- expect(cluster.getService(SERVICE_NAME)).andReturn(service);
- expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent);
- expect(sch.getServiceName()).andReturn(SERVICE_NAME);
- expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME);
+
+ expect(cluster.getService(SERVICE_NAME)).andReturn(service).atLeastOnce();
+ expect(service.getServiceComponent(SERVICE_COMPONENT_NAME)).andReturn(serviceComponent).atLeastOnce();
+ expect(sch.getServiceName()).andReturn(SERVICE_NAME).atLeastOnce();
+ expect(sch.getServiceComponentName()).andReturn(SERVICE_COMPONENT_NAME).atLeastOnce();
}
@Test
@@ -269,7 +301,6 @@ public class StackVersionListenerTest extends EasyMockSupport {
String newVersion = VALID_NEW_VERSION;
HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, sch, newVersion, 1L);
- StackVersionListener listener = new StackVersionListener(publisher);
// !!! avoid injector for test class
Field field = StackVersionListener.class.getDeclaredField("repositoryVersionDAO");
field.setAccessible(true);
@@ -298,7 +329,6 @@ public class StackVersionListenerTest extends EasyMockSupport {
replayAll();
// !!! avoid injector for test class
- StackVersionListener listener = new StackVersionListener(publisher);
Field field = StackVersionListener.class.getDeclaredField("repositoryVersionDAO");
field.setAccessible(true);
@@ -317,7 +347,6 @@ public class StackVersionListenerTest extends EasyMockSupport {
private void sendEventAndVerify(String newVersion) {
HostComponentVersionAdvertisedEvent event = new HostComponentVersionAdvertisedEvent(cluster, sch, newVersion);
- StackVersionListener listener = new StackVersionListener(publisher);
listener.onAmbariEvent(event);
verifyAll();
[04/16] ambari git commit: AMBARI-18698. Filter by roles in Users
List page takes upto 20 secs to load with 1000+ users. (mpapirkovskyy)
Posted by nc...@apache.org.
AMBARI-18698. Filter by roles in Users List page takes upto 20 secs to load with 1000+ users. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03f61bd6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03f61bd6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03f61bd6
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 03f61bd6a10be892d67237c45f1db34830d4c5d8
Parents: aa588ca
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Wed Oct 26 14:40:13 2016 +0300
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Wed Oct 26 20:57:46 2016 +0300
----------------------------------------------------------------------
.../internal/UserPrivilegeResourceProvider.java | 116 +++++++++++++++++--
.../UserPrivilegeResourceProviderTest.java | 2 +
2 files changed, 109 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/03f61bd6/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
index 009c38b..ba32a5f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProvider.java
@@ -17,6 +17,9 @@
*/
package org.apache.ambari.server.controller.internal;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
import org.apache.ambari.server.controller.spi.Predicate;
@@ -30,6 +33,7 @@ import org.apache.ambari.server.orm.dao.UserDAO;
import org.apache.ambari.server.orm.dao.ViewInstanceDAO;
import org.apache.ambari.server.orm.entities.ClusterEntity;
import org.apache.ambari.server.orm.entities.GroupEntity;
+import org.apache.ambari.server.orm.entities.PrincipalEntity;
import org.apache.ambari.server.orm.entities.PrincipalTypeEntity;
import org.apache.ambari.server.orm.entities.PrivilegeEntity;
import org.apache.ambari.server.orm.entities.UserEntity;
@@ -48,6 +52,8 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
/**
* Resource provider for user privilege resources.
@@ -142,6 +148,87 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
keyPropertyIds.put(Resource.Type.UserPrivilege, PRIVILEGE_PRIVILEGE_ID_PROPERTY_ID);
}
+ private ThreadLocal<LoadingCache<Long, ClusterEntity>> clusterCache =
+ new ThreadLocal<LoadingCache<Long, ClusterEntity>>(){
+ @Override
+ protected LoadingCache<Long, ClusterEntity> initialValue() {
+ CacheLoader<Long, ClusterEntity> loader = new CacheLoader<Long, ClusterEntity>() {
+ @Override
+ public ClusterEntity load(Long key) throws Exception {
+ return clusterDAO.findByResourceId(key);
+ }
+ };
+ return CacheBuilder.newBuilder().expireAfterWrite(20, TimeUnit.SECONDS).build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<Long, ViewInstanceEntity>> viewInstanceCache =
+ new ThreadLocal<LoadingCache<Long, ViewInstanceEntity>>(){
+ @Override
+ protected LoadingCache<Long, ViewInstanceEntity> initialValue() {
+ CacheLoader<Long, ViewInstanceEntity> loader = new CacheLoader<Long, ViewInstanceEntity>() {
+ @Override
+ public ViewInstanceEntity load(Long key) throws Exception {
+ return viewInstanceDAO.findByResourceId(key);
+ }
+ };
+ return CacheBuilder.newBuilder().expireAfterWrite(20, TimeUnit.SECONDS).build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<String, UserEntity>> usersCache =
+ new ThreadLocal<LoadingCache<String, UserEntity>>(){
+ @Override
+ protected LoadingCache<String, UserEntity> initialValue() {
+ CacheLoader<String, UserEntity> loader = new CacheLoader<String, UserEntity>() {
+ @Override
+ public UserEntity load(String key) throws Exception {
+ //fallback mechanism, mostly for unit tests
+ UserEntity userEntity = userDAO.findLocalUserByName(key);
+ if (userEntity == null) {
+ userEntity = userDAO.findLdapUserByName(key);
+ }
+ if (userEntity == null) {
+ userEntity = userDAO.findUserByNameAndType(key, UserType.JWT);
+ }
+ return userEntity;
+ }
+ };
+
+ return CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .build(loader);
+ }
+ };
+
+ private ThreadLocal<LoadingCache<PrincipalEntity, GroupEntity>> groupsCache =
+ new ThreadLocal<LoadingCache<PrincipalEntity, GroupEntity>>(){
+ @Override
+ protected LoadingCache<PrincipalEntity, GroupEntity> initialValue() {
+ CacheLoader<PrincipalEntity, GroupEntity> loader = new CacheLoader<PrincipalEntity, GroupEntity>() {
+ @Override
+ public GroupEntity load(PrincipalEntity key) throws Exception {
+ return groupDAO.findGroupByPrincipal(key);
+ }
+ };
+
+ return CacheBuilder.newBuilder()
+ .expireAfterWrite(20, TimeUnit.SECONDS)
+ .build(loader);
+ }
+ };
+
+ private GroupEntity getCachedGroupByPrincipal(PrincipalEntity principalEntity) {
+ GroupEntity entity = groupsCache.get().getIfPresent(principalEntity);
+ if (entity == null) {
+ for (GroupEntity groupEntity : groupDAO.findAll()) {
+ groupsCache.get().put(groupEntity.getPrincipal(), groupEntity);
+ }
+ entity = groupsCache.get().getUnchecked(principalEntity);
+ }
+ return entity;
+ }
+
/**
* Constructor.
@@ -183,13 +270,24 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
}
if (userName != null) {
- UserEntity userEntity = userDAO.findLocalUserByName(userName);
- if (userEntity == null) {
- userEntity = userDAO.findLdapUserByName(userName);
- }
+
+ UserEntity userEntity = usersCache.get().getIfPresent(userName);
if (userEntity == null) {
- userEntity = userDAO.findUserByNameAndType(userName, UserType.JWT);
+ //temporary tradeoff, add ~200ms for single user call, but start saving time for 100+ subsequent calls
+ //usual case for management page is to populate subresources for all users
+ Map<String, UserEntity> userNames = new TreeMap<>();
+ for (UserEntity entity : userDAO.findAll()) {
+ UserEntity existing = userNames.get(entity.getUserName());
+ if (existing == null ||
+ entity.getUserType() == UserType.LOCAL ||
+ existing.getUserType() == UserType.JWT) {
+ userNames.put(entity.getUserName(), entity);
+ }
+ }
+ usersCache.get().putAll(userNames);
+ userEntity = usersCache.get().getUnchecked(userName);
}
+
if (userEntity == null) {
throw new SystemException("User " + userName + " was not found");
}
@@ -213,7 +311,7 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
* @param requestedIds the relevant request ids
* @return a resource
*/
- protected Resource toResource(PrivilegeEntity privilegeEntity, Object userName, Set<String> requestedIds) {
+ protected Resource toResource(PrivilegeEntity privilegeEntity, Object userName, Set<String> requestedIds){
final ResourceImpl resource = new ResourceImpl(Resource.Type.UserPrivilege);
setResourceProperty(resource, PRIVILEGE_USER_NAME_PROPERTY_ID, userName, requestedIds);
@@ -227,7 +325,7 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
final UserEntity user = userDAO.findUserByPrincipal(privilegeEntity.getPrincipal());
setResourceProperty(resource, PRIVILEGE_PRINCIPAL_NAME_PROPERTY_ID, user.getUserName(), requestedIds);
} else if (principalTypeName.equals(PrincipalTypeEntity.GROUP_PRINCIPAL_TYPE_NAME)) {
- final GroupEntity groupEntity = groupDAO.findGroupByPrincipal(privilegeEntity.getPrincipal());
+ final GroupEntity groupEntity = getCachedGroupByPrincipal(privilegeEntity.getPrincipal());
setResourceProperty(resource, PRIVILEGE_PRINCIPAL_NAME_PROPERTY_ID, groupEntity.getGroupName(), requestedIds);
}
@@ -239,11 +337,11 @@ public class UserPrivilegeResourceProvider extends ReadOnlyResourceProvider {
// there is nothing special to add for this case
break;
case CLUSTER:
- final ClusterEntity clusterEntity = clusterDAO.findByResourceId(privilegeEntity.getResource().getId());
+ final ClusterEntity clusterEntity = clusterCache.get().getUnchecked(privilegeEntity.getResource().getId());
setResourceProperty(resource, PRIVILEGE_CLUSTER_NAME_PROPERTY_ID, clusterEntity.getClusterName(), requestedIds);
break;
case VIEW:
- final ViewInstanceEntity viewInstanceEntity = viewInstanceDAO.findByResourceId(privilegeEntity.getResource().getId());
+ final ViewInstanceEntity viewInstanceEntity = viewInstanceCache.get().getUnchecked(privilegeEntity.getResource().getId());
final ViewEntity viewEntity = viewInstanceEntity.getViewEntity();
setResourceProperty(resource, PRIVILEGE_VIEW_NAME_PROPERTY_ID, viewEntity.getCommonName(), requestedIds);
http://git-wip-us.apache.org/repos/asf/ambari/blob/03f61bd6/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
index ddb510d..ce2d8e1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UserPrivilegeResourceProviderTest.java
@@ -385,6 +385,7 @@ public class UserPrivilegeResourceProviderTest extends AbstractPrivilegeResource
final UserDAO userDAO = createNiceMock(UserDAO.class);
expect(userDAO.findLocalUserByName("jdoe")).andReturn(userEntity).anyTimes();
expect(userDAO.findUserByPrincipal(anyObject(PrincipalEntity.class))).andReturn(userEntity).anyTimes();
+ expect(userDAO.findAll()).andReturn(Collections.<UserEntity>emptyList()).anyTimes();
final PrivilegeDAO privilegeDAO = createMock(PrivilegeDAO.class);
final MemberDAO memberDAO = createMock(MemberDAO.class);
@@ -465,6 +466,7 @@ public class UserPrivilegeResourceProviderTest extends AbstractPrivilegeResource
andReturn(Collections.<MemberEntity>emptyList())
.atLeastOnce();
expect(userDAO.findLocalUserByName(requestedUsername)).andReturn(userEntity).anyTimes();
+ expect(userDAO.findAll()).andReturn(Collections.<UserEntity>emptyList()).anyTimes();
expect(userEntity.getPrincipal()).andReturn(principalEntity).anyTimes();
expect(userEntity.getMemberEntities()).andReturn(Collections.<MemberEntity>emptySet()).anyTimes();
expect(privilegeEntity.getPermission()).andReturn(permissionEntity).anyTimes();
[10/16] ambari git commit: AMBARI-18669 - Manage Journal Node Wizard
Initial UI (rzang)
Posted by nc...@apache.org.
AMBARI-18669 - Manage Journal Node Wizard Initial UI <fix_style> (rzang)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d52154ee
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d52154ee
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d52154ee
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: d52154ee1e526efb78b1847106d0ea85462d596b
Parents: 0c5bcbb
Author: Richard Zang <rz...@apache.org>
Authored: Wed Oct 26 18:08:52 2016 -0700
Committer: Richard Zang <rz...@apache.org>
Committed: Wed Oct 26 18:08:52 2016 -0700
----------------------------------------------------------------------
.../app/routes/manage_journalnode_routes.js | 3 +-
.../highAvailability/journalNode/wizard.hbs | 34 ++++++++------------
2 files changed, 15 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d52154ee/ambari-web/app/routes/manage_journalnode_routes.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/routes/manage_journalnode_routes.js b/ambari-web/app/routes/manage_journalnode_routes.js
index b108f74..4803ca7 100644
--- a/ambari-web/app/routes/manage_journalnode_routes.js
+++ b/ambari-web/app/routes/manage_journalnode_routes.js
@@ -26,7 +26,8 @@ module.exports = App.WizardRoute.extend({
manageJournalNodeWizardController.dataLoading().done(function () {
App.router.get('updateController').set('isWorking', false);
var popup = App.ModalPopup.show({
- classNames: ['full-width-modal'],
+ classNames: ['wizard-modal-wrapper'],
+ modalDialogClasses: ['modal-xlg'],
header: Em.I18n.t('admin.manageJournalNode.wizard.header'),
bodyClass: App.ManageJournalNodeWizardView.extend({
controller: manageJournalNodeWizardController
http://git-wip-us.apache.org/repos/asf/ambari/blob/d52154ee/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs b/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
index a15b8c0..c191d50 100644
--- a/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
+++ b/ambari-web/app/templates/main/admin/highAvailability/journalNode/wizard.hbs
@@ -17,27 +17,19 @@
}}
<div class="wizard">
- <div class="container">
- <div class="container-fluid">
- <div class="row-fluid">
- <div class="span3">
- <!--Sidebar content-->
- <div class="well">
- <ul class="nav nav-pills nav-stacked">
- <li class="nav-header">{{t admin.manageJournalNode.wizard.header}}</li>
- <li {{bindAttr class="isStep1:active view.isStep1Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep1 target="controller"}}>{{t admin.manageJournalNode.wizard.step1.header}}</a></li>
- <li {{bindAttr class="isStep2:active view.isStep2Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep2 target="controller"}}>{{t admin.manageJournalNode.wizard.step2.header}}</a></li>
- <li {{bindAttr class="isStep3:active view.isStep3Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep3 target="controller"}}>{{t admin.manageJournalNode.wizard.step3.header}}</a></li>
- <li {{bindAttr class="isStep4:active view.isStep4Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep4 target="controller"}}>{{t admin.manageJournalNode.wizard.step4.header}}</a></li>
- <li {{bindAttr class="isStep5:active view.isStep5Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}>{{t admin.manageJournalNode.wizard.step5.header}}</a></li>
- <li {{bindAttr class="isStep6:active view.isStep6Disabled:disabled"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}>{{t admin.manageJournalNode.wizard.step6.header}}</a></li>
- </ul>
- </div>
- </div>
- <div class="wizard-content well span9">
- {{outlet}}
- </div>
- </div>
+ <div class="wizard-body row">
+ <div class="wizard-nav col-md-3">
+ <ul class="nav nav-pills nav-stacked">
+ <li {{bindAttr class="isStep1:active view.isStep1Disabled:disabled view.isStep1Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep1 target="controller"}}><i class="step-marker"><span class="step-index">1</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step1.header}}</p></a></li>
+ <li {{bindAttr class="isStep2:active view.isStep2Disabled:disabled view.isStep2Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep2 target="controller"}}><i class="step-marker"><span class="step-index">2</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step2.header}}</p></a></li>
+ <li {{bindAttr class="isStep3:active view.isStep3Disabled:disabled view.isStep3Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep3 target="controller"}}><i class="step-marker"><span class="step-index">3</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step3.header}}</p></a></li>
+ <li {{bindAttr class="isStep4:active view.isStep4Disabled:disabled view.isStep4Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep4 target="controller"}}><i class="step-marker"><span class="step-index">4</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step4.header}}</p></a></li>
+ <li {{bindAttr class="isStep5:active view.isStep5Disabled:disabled view.isStep5Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep5 target="controller"}}><i class="step-marker"><span class="step-index">5</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step5.header}}</p></a></li>
+ <li {{bindAttr class="isStep6:active view.isStep6Disabled:disabled view.isStep6Completed:completed"}}><a href="javascript:void(null);" {{action gotoStep6 target="controller"}}><i class="step-marker"><span class="step-index">6</span></i><p class="step-name">{{t admin.manageJournalNode.wizard.step6.header}}</p></a></li>
+ </ul>
+ </div>
+ <div class="wizard-content col-md-9">
+ {{outlet}}
</div>
</div>
</div>
[05/16] ambari git commit: AMBARI-18695: Add PXF Hive ORC Profile
(mithmatt)
Posted by nc...@apache.org.
AMBARI-18695: Add PXF Hive ORC Profile (mithmatt)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/93a3fe1e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/93a3fe1e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/93a3fe1e
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 93a3fe1e88b7e7d7361ed7c5399ab41b985ea694
Parents: 03f61bd
Author: Matt <mm...@pivotal.io>
Authored: Wed Oct 26 11:29:17 2016 -0700
Committer: Matt <mm...@pivotal.io>
Committed: Wed Oct 26 11:29:17 2016 -0700
----------------------------------------------------------------------
.../PXF/3.0.0/configuration/pxf-profiles.xml | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/93a3fe1e/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml
index 3c2aac0..77aa4fc 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-profiles.xml
@@ -107,6 +107,20 @@ under the License.
</plugins>
</profile>
<profile>
+ <name>HiveORC</name>
+ <description>This profile is suitable only for Hive tables stored in ORC files
+ and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe.
+ It is much faster than the general purpose Hive profile.
+ DELIMITER parameter is mandatory.
+ </description>
+ <plugins>
+ <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
+ <accessor>org.apache.hawq.pxf.plugins.hive.HiveORCAccessor</accessor>
+ <resolver>org.apache.hawq.pxf.plugins.hive.HiveORCSerdeResolver</resolver>
+ <metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
+ </plugins>
+ </profile>
+ <profile>
<name>HdfsTextSimple</name>
<description>This profile is suitable for using when reading delimited single line records from plain text files
on HDFS
[07/16] ambari git commit: AMBARI-18666: Move HAWQ and PXF RCO from
stacks to common-services (mithmatt)
Posted by nc...@apache.org.
AMBARI-18666: Move HAWQ and PXF RCO from stacks to common-services (mithmatt)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/039562ca
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/039562ca
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/039562ca
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 039562ca1f7b76222b574062da9756bae72d9a15
Parents: ae6e204
Author: Matt <mm...@pivotal.io>
Authored: Wed Oct 26 14:59:48 2016 -0700
Committer: Matt <mm...@pivotal.io>
Committed: Wed Oct 26 14:59:48 2016 -0700
----------------------------------------------------------------------
.../common-services/HAWQ/2.0.0/role_command_order.json | 11 +++++++++++
.../common-services/PXF/3.0.0/role_command_order.json | 7 +++++++
.../resources/stacks/HDP/2.3/role_command_order.json | 8 --------
3 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/039562ca/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/role_command_order.json b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/role_command_order.json
new file mode 100644
index 0000000..6b6f7c2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/role_command_order.json
@@ -0,0 +1,11 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for HAWQ",
+ "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START", "NODEMANAGER-START"],
+ "HAWQSTANDBY-START" : ["HAWQMASTER-START"],
+ "HAWQSTANDBY-RESTART" : ["HAWQMASTER-RESTART"],
+ "HAWQSEGMENT-START" : ["HAWQMASTER-START", "HAWQSTANDBY-START"],
+ "HAWQSEGMENT-RESTART" : ["HAWQMASTER-RESTART", "HAWQSTANDBY-RESTART"],
+ "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQSEGMENT-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK", "YARN_SERVICE_CHECK-SERVICE_CHECK", "PXF_SERVICE_CHECK-SERVICE_CHECK"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/039562ca/ambari-server/src/main/resources/common-services/PXF/3.0.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/role_command_order.json b/ambari-server/src/main/resources/common-services/PXF/3.0.0/role_command_order.json
new file mode 100644
index 0000000..a14aa49
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for PXF",
+ "PXF-START" : ["NAMENODE-START", "DATANODE-START"],
+ "PXF_SERVICE_CHECK-SERVICE_CHECK" : ["PXF-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK", "HBASE_SERVICE_CHECK-SERVICE_CHECK", "HIVE_SERVICE_CHECK-SERVICE_CHECK"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/039562ca/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index edd6d64..2c841ca 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -12,14 +12,6 @@
"ATLAS_SERVER-START": ["KAFKA_BROKER-START", "INFRA_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
"SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],
"RESOURCEMANAGER-STOP" : ["SPARK_THRIFTSERVER-STOP"],
- "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START", "NODEMANAGER-START"],
- "HAWQSTANDBY-START" : ["HAWQMASTER-START"],
- "HAWQSTANDBY-RESTART" : ["HAWQMASTER-RESTART"],
- "HAWQSEGMENT-START" : ["HAWQMASTER-START", "HAWQSTANDBY-START"],
- "HAWQSEGMENT-RESTART" : ["HAWQMASTER-RESTART", "HAWQSTANDBY-RESTART"],
- "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQSEGMENT-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK", "YARN_SERVICE_CHECK-SERVICE_CHECK", "PXF_SERVICE_CHECK-SERVICE_CHECK"],
- "PXF-START" : ["NAMENODE-START", "DATANODE-START"],
- "PXF_SERVICE_CHECK-SERVICE_CHECK" : ["PXF-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK", "HBASE_SERVICE_CHECK-SERVICE_CHECK", "HIVE_SERVICE_CHECK-SERVICE_CHECK"],
"KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],
"KAFKA_BROKER-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
"NIMBUS-START" : ["ZOOKEEPER_SERVER-START", "RANGER_USERSYNC-START", "NAMENODE-START"],
[09/16] ambari git commit: AMBARI-18705 : All host metrics not being
collected by AMS if user does not have permissions to read mount point.
(avijayan)
Posted by nc...@apache.org.
AMBARI-18705 : All host metrics not being collected by AMS if user does not have permissions to read mount point. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0c5bcbba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0c5bcbba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0c5bcbba
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 0c5bcbba5d44e915a5487a855f364bfe8cab6b72
Parents: 00ce02a
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Wed Oct 26 16:57:34 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Wed Oct 26 16:57:34 2016 -0700
----------------------------------------------------------------------
.../src/main/python/core/host_info.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c5bcbba/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
index 632c86b..7992cff 100644
--- a/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
+++ b/ambari-metrics/ambari-metrics-host-monitoring/src/main/python/core/host_info.py
@@ -178,6 +178,7 @@ class HostInfo():
max_percent_usage = ('', 0)
partition_count = 0
+ devices = set()
for part in psutil.disk_partitions(all=False):
if os.name == 'nt':
if 'cdrom' in part.opts or part.fstype == '':
@@ -187,8 +188,15 @@ class HostInfo():
continue
pass
pass
- usage = psutil.disk_usage(part.mountpoint)
-
+ try:
+ usage = psutil.disk_usage(part.mountpoint)
+ except Exception, e:
+ logger.error('Failed to read disk_usage for a mountpoint : ' + str(e))
+ continue
+
+ if part.device in devices: # Skip devices already seen.
+ continue
+ devices.add(part.device)
combined_disk_total += usage.total if hasattr(usage, 'total') else 0
combined_disk_used += usage.used if hasattr(usage, 'used') else 0
combined_disk_free += usage.free if hasattr(usage, 'free') else 0
[08/16] ambari git commit: AMBARI-18707. Css improvement: Error
tooltip context doesn't fit tooltip view on install Step1 (alexantonenko)
Posted by nc...@apache.org.
AMBARI-18707. Css improvement: Error tooltip context doesn't fit tooltip view on install Step1 (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/00ce02a5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/00ce02a5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/00ce02a5
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 00ce02a5f0786f827e308edc28b8aa3810ad7468
Parents: 039562c
Author: Alex Antonenko <hi...@gmail.com>
Authored: Wed Oct 26 22:02:18 2016 +0300
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Thu Oct 27 02:53:07 2016 +0300
----------------------------------------------------------------------
ambari-web/app/styles/application.less | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/00ce02a5/ambari-web/app/styles/application.less
----------------------------------------------------------------------
diff --git a/ambari-web/app/styles/application.less b/ambari-web/app/styles/application.less
index d65ca9d..a33b6bb 100644
--- a/ambari-web/app/styles/application.less
+++ b/ambari-web/app/styles/application.less
@@ -70,6 +70,11 @@ select.form-control {
display: inline-block;
}
+// Override bootstrap class to ensure that popover content will fit to popover window
+.popover-content {
+ word-break: break-word;
+}
+
#wrapper {
min-height: 100%;
}
[12/16] ambari git commit: AMBARI-18712. compare functionality
between config versions is not working (onechiporenko)
Posted by nc...@apache.org.
AMBARI-18712. compare functionality between config versions is not working (onechiporenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c6809ff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c6809ff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c6809ff
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 9c6809ff599c10f4e6d3311df9792580b74bf6c8
Parents: 4b09b63
Author: Oleg Nechiporenko <on...@apache.org>
Authored: Thu Oct 27 12:59:56 2016 +0300
Committer: Oleg Nechiporenko <on...@apache.org>
Committed: Thu Oct 27 13:55:13 2016 +0300
----------------------------------------------------------------------
ambari-web/app/views/common/configs/config_history_flow.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9c6809ff/ambari-web/app/views/common/configs/config_history_flow.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/configs/config_history_flow.js b/ambari-web/app/views/common/configs/config_history_flow.js
index 58de1e6..931d994 100644
--- a/ambari-web/app/views/common/configs/config_history_flow.js
+++ b/ambari-web/app/views/common/configs/config_history_flow.js
@@ -326,8 +326,8 @@ App.ConfigHistoryFlowView = Em.View.extend({
* add a second version-info-bar for the chosen version
*/
compare: function (event) {
- this.set('controller.compareServiceVersion', this.get('hoveredServiceVersion'));
var serviceConfigVersion = this.get('hoveredServiceVersion') || event.context;
+ this.set('controller.compareServiceVersion', serviceConfigVersion);
this.set('compareServiceVersion', serviceConfigVersion);
var controller = this.get('controller');
[06/16] ambari git commit: AMBARI-18709 - Javascript Error When Using
ComponentState Filter On Host Page (rzang)
Posted by nc...@apache.org.
AMBARI-18709 - Javascript Error When Using ComponentState Filter On Host Page (rzang)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ae6e2047
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ae6e2047
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ae6e2047
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: ae6e204732af54780f9177c840ef8649b48ba84a
Parents: 93a3fe1
Author: Richard Zang <rz...@apache.org>
Authored: Wed Oct 26 14:10:00 2016 -0700
Committer: Richard Zang <rz...@apache.org>
Committed: Wed Oct 26 14:10:00 2016 -0700
----------------------------------------------------------------------
ambari-web/app/controllers/main/host/combo_search_box.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ae6e2047/ambari-web/app/controllers/main/host/combo_search_box.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/host/combo_search_box.js b/ambari-web/app/controllers/main/host/combo_search_box.js
index 4ab3b7c..dedd290 100644
--- a/ambari-web/app/controllers/main/host/combo_search_box.js
+++ b/ambari-web/app/controllers/main/host/combo_search_box.js
@@ -74,7 +74,7 @@ App.MainHostComboSearchBoxController = Em.Controller.extend({
createComboParamURL: function(pHash, expressions) {
var self = this;
var result = '';
- for (key in pHash) {
+ for (var key in pHash) {
var v = pHash[key];
if (Em.isArray(v)) {
var ex = '(';
[02/16] ambari git commit: AMBARI-18699 - Upgrade Configuration Packs
Should Have an XSD (jonathanhurley)
Posted by nc...@apache.org.
AMBARI-18699 - Upgrade Configuration Packs Should Have an XSD (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7b30be6d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7b30be6d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7b30be6d
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 7b30be6dd49a61593c198306eb2feefbd85a680b
Parents: a5b8230
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Oct 26 08:18:10 2016 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Oct 26 12:44:12 2016 -0400
----------------------------------------------------------------------
.../upgrade/ConfigUpgradeChangeDefinition.java | 75 +--------
.../state/stack/upgrade/ConfigureTask.java | 49 +-----
.../stacks/HDP/2.2/upgrades/config-upgrade.xml | 110 +++++--------
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 28 ++--
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 29 ++--
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 4 +-
.../src/main/resources/upgrade-config.xsd | 163 +++++++++++++++++++
.../ambari/server/state/UpgradeHelperTest.java | 50 +++---
.../state/stack/ConfigUpgradeValidityTest.java | 68 +++++++-
.../HDP/2.1.1/upgrades/config-upgrade.xml | 52 ++----
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 2 +-
.../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +--
.../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +--
13 files changed, 353 insertions(+), 315 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
index 54431eb..5428ea7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
@@ -17,10 +17,9 @@
*/
package org.apache.ambari.server.state.stack.upgrade;
-import com.google.gson.Gson;
-import org.apache.ambari.server.AmbariException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -28,9 +27,11 @@ import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
/**
* The {@link ConfigUpgradeChangeDefinition} represents a configuration change. This change can be
@@ -123,9 +124,6 @@ public class ConfigUpgradeChangeDefinition {
@XmlElement(name = "set")
private List<ConfigurationKeyValue> keyValuePairs;
- @XmlElement(name = "condition")
- private List<Condition> conditions;
-
@XmlElement(name = "transfer")
private List<Transfer> transfers;
@@ -147,13 +145,6 @@ public class ConfigUpgradeChangeDefinition {
}
/**
- * @return the list of conditions
- */
- public List<Condition> getConditions() {
- return conditions;
- }
-
- /**
* @return the list of transfers, checking for appropriate null fields.
*/
public List<Transfer> getTransfers() {
@@ -267,56 +258,6 @@ public class ConfigUpgradeChangeDefinition {
}
/**
- * A conditional element that will only perform the configuration if the
- * condition is met.
- */
- @XmlAccessorType(XmlAccessType.FIELD)
- @XmlType(name = "condition")
- public static class Condition {
- @XmlAttribute(name = "type")
- private String conditionConfigType;
-
- @XmlAttribute(name = "key")
- private String conditionKey;
-
- @XmlAttribute(name = "value")
- private String conditionValue;
-
- @XmlElement(name = "type")
- private String configType;
-
- @XmlElement(name = "key")
- private String key;
-
- @XmlElement(name = "value")
- private String value;
-
- public String getConditionConfigType() {
- return conditionConfigType;
- }
-
- public String getConditionKey() {
- return conditionKey;
- }
-
- public String getConditionValue() {
- return conditionValue;
- }
-
- public String getConfigType() {
- return configType;
- }
-
- public String getKey() {
- return key;
- }
-
- public String getValue() {
- return value;
- }
- }
-
- /**
* A {@code transfer} element will copy, move, or delete the value of one type/key to another type/key.
*/
@XmlAccessorType(XmlAccessType.FIELD)
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index a9b355a..d7bb338 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -33,7 +33,6 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
-import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Condition;
import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue;
import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Replace;
import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Transfer;
@@ -192,40 +191,6 @@ public class ConfigureTask extends ServerSideActionTask {
return configParameters;
}
- // the first matched condition will win; conditions make configuration tasks singular in
- // the properties that can be set - when there is a condition the task will only contain
- // conditions
- List<Condition> conditions = definition.getConditions();
- if( null != conditions && !conditions.isEmpty() ){
- for (Condition condition : conditions) {
- String conditionConfigType = condition.getConditionConfigType();
- String conditionKey = condition.getConditionKey();
- String conditionValue = condition.getConditionValue();
-
- // always add the condition's target type just so that we have one to
- // return even if none of the conditions match
- configParameters.put(PARAMETER_CONFIG_TYPE, condition.getConfigType());
-
- // check the condition; if it passes, set the configuration properties
- // and break
- String checkValue = getDesiredConfigurationValue(cluster,
- conditionConfigType, conditionKey);
-
- if (conditionValue.equals(checkValue)) {
- List<ConfigurationKeyValue> configurations = new ArrayList<>(1);
- ConfigurationKeyValue keyValue = new ConfigurationKeyValue();
- keyValue.key = condition.getKey();
- keyValue.value = condition.getValue();
- configurations.add(keyValue);
-
- configParameters.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS,
- m_gson.toJson(configurations));
-
- return configParameters;
- }
- }
- }
-
// this task is not a condition task, so process the other elements normally
if (null != definition.getConfigType()) {
configParameters.put(PARAMETER_CONFIG_TYPE, definition.getConfigType());
@@ -260,8 +225,9 @@ public class ConfigureTask extends ServerSideActionTask {
for(Replace replacement: replacements){
if(isValidConditionSettings(cluster, configType, replacement.key,
- replacement.ifKey, replacement.ifType, replacement.ifValue, replacement.ifKeyState))
+ replacement.ifKey, replacement.ifType, replacement.ifValue, replacement.ifKeyState)) {
allowedReplacements.add(replacement);
+ }
}
return allowedReplacements;
@@ -272,8 +238,9 @@ public class ConfigureTask extends ServerSideActionTask {
for(ConfigurationKeyValue configurationKeyValue: sets){
if(isValidConditionSettings(cluster, configType, configurationKeyValue.key,
- configurationKeyValue.ifKey, configurationKeyValue.ifType, configurationKeyValue.ifValue, configurationKeyValue.ifKeyState))
+ configurationKeyValue.ifKey, configurationKeyValue.ifType, configurationKeyValue.ifValue, configurationKeyValue.ifKeyState)) {
allowedSets.add(configurationKeyValue);
+ }
}
return allowedSets;
@@ -283,14 +250,16 @@ public class ConfigureTask extends ServerSideActionTask {
List<Transfer> allowedTransfers = new ArrayList<>();
for (Transfer transfer : transfers) {
String key = "";
- if(transfer.operation == TransferOperation.DELETE)
+ if(transfer.operation == TransferOperation.DELETE) {
key = transfer.deleteKey;
- else
+ } else {
key = transfer.fromKey;
+ }
if(isValidConditionSettings(cluster, configType, key,
- transfer.ifKey, transfer.ifType, transfer.ifValue, transfer.ifKeyState))
+ transfer.ifKey, transfer.ifType, transfer.ifValue, transfer.ifKeyState)) {
allowedTransfers.add(transfer);
+ }
}
return allowedTransfers;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
index c225cca..6af8b43 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
@@ -16,8 +16,7 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="HDFS">
<component name="NAMENODE">
@@ -54,11 +53,12 @@
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_hdfs_adjust_ranger_plugin">
- <condition type="ranger-hdfs-plugin-properties" key="ranger-hdfs-plugin-enabled" value="Yes">
- <type>hdfs-site</type>
- <key>dfs.namenode.inode.attributes.provider.class</key>
- <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
- </condition>
+ <type>hdfs-site</type>
+ <set key="dfs.namenode.inode.attributes.provider.class"
+ value="org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer"
+ if-type="ranger-hdfs-plugin-properties"
+ if-key="ranger-hdfs-plugin-enabled"
+ if-value="Yes"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_hdfs_transition_ranger_hdfs_policy"
@@ -446,16 +446,9 @@
</definition>
<definition xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties">
- <condition type="tez-site" key="tez.counters.max" value="2000">
- <type>tez-site</type>
- <key>tez.counters.max</key>
- <value>10000</value>
- </condition>
- <condition type="tez-site" key="tez.counters.max.groups" value="1000">
- <type>tez-site</type>
- <key>tez.counters.max.groups</key>
- <value>3000</value>
- </condition>
+ <type>tez-site</type>
+ <set key="tez.counters.max" value="10000" if-type="tez-site" if-key="tez.counters.max" if-value="2000"/>
+ <set key="tez.counters.max.groups" value="3000" if-type="tez-site" if-key="tez.counters.max.groups" if-value="1000"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties">
@@ -493,11 +486,8 @@
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_nimbus_monitor_freq_adjustment">
- <condition type="storm-site" key="nimbus.monitor.freq.secs" value="10">
- <type>storm-site</type>
- <key>nimbus.monitor.freq.secs</key>
- <value>120</value>
- </condition>
+ <type>storm-site</type>
+ <set key="nimbus.monitor.freq.secs" value="120" if-type="storm-site" if-key="nimbus.monitor.freq.secs" if-value="10"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_nimbus_convert_nimbus_host_to_seeds"
@@ -699,19 +689,13 @@
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin_hdfs_audit">
- <condition type="ranger-env" key="xasecure.audit.destination.hdfs" value="false">
- <type>ranger-env</type>
- <key>xasecure.audit.destination.hdfs</key>
- <value>false</value>
- </condition>
+ <type>ranger-env</type>
+ <set key="xasecure.audit.destination.hdfs" value="false" if-type="ranger-env" if-key="xasecure.audit.destination.hdfs" if-value="false"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_admin_db_audit">
- <condition type="ranger-env" key="xasecure.audit.destination.db" value="true">
- <type>ranger-env</type>
- <key>xasecure.audit.destination.db</key>
- <value>true</value>
- </condition>
+ <type>ranger-env</type>
+ <set key="xasecure.audit.destination.db" value="true" if-type="ranger-env" if-key="xasecure.audit.destination.db" if-value="true"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync" summary="Updating Ranger Usersync">
@@ -753,17 +737,14 @@
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_sync_source">
- <condition type="usersync-properties" key="SYNC_SOURCE" value="unix">
- <type>ranger-ugsync-site</type>
- <key>ranger.usersync.source.impl.class</key>
- <value>org.apache.ranger.unixusersync.process.UnixUserGroupBuilder</value>
- </condition>
+ <type>ranger-ugsync-site</type>
+ <set key="ranger.usersync.source.impl.class"
+ value="org.apache.ranger.unixusersync.process.UnixUserGroupBuilder"
+ if-type="usersync-properties" if-key="SYNC_SOURCE" if-value="unix"/>
- <condition type="usersync-properties" key="SYNC_SOURCE" value="ldap">
- <type>ranger-ugsync-site</type>
- <key>ranger.usersync.source.impl.class</key>
- <value>org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder</value>
- </condition>
+ <set key="ranger.usersync.source.impl.class"
+ value="org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder"
+ if-type="usersync-properties" if-key="SYNC_SOURCE" if-value="ldap"/>
</definition>
<definition xsi:type="configure" id="hdp_2_3_0_0_update_ranger_usersync_properties">
@@ -793,25 +774,18 @@
<service name="HBASE">
<component name="HBASE_MASTER">
<changes>
- <definition xsi:type="configure"
- id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory">
- <condition type="hbase-env" key="phoenix_sql_enabled" value="true">
- <type>hbase-site</type>
- <key>hbase.region.server.rpc.scheduler.factory.class</key>
- <value>org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory
- </value>
- </condition>
+ <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory">
+ <type>hbase-site</type>
+ <set key="hbase.region.server.rpc.scheduler.factory.class"
+ value="org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory"
+ if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/>
</definition>
- <definition xsi:type="configure"
- id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory">
- <condition type="hbase-env" key="phoenix_sql_enabled" value="true">
- <type>hbase-site</type>
- <key>hbase.rpc.controllerfactory.class</key>
- <value>
- org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory
- </value>
- </condition>
+ <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_rpc_controller_factory">
+ <type>hbase-site</type>
+ <set key="hbase.rpc.controllerfactory.class"
+ value="org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory"
+ if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/>
</definition>
<definition xsi:type="configure"
@@ -823,15 +797,11 @@
default-value="0.4"/>
</definition>
- <definition xsi:type="configure"
- id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec">
- <condition type="hbase-env" key="phoenix_sql_enabled" value="true">
- <type>hbase-site</type>
- <key>hbase.regionserver.wal.codec</key>
- <value>
- org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec
- </value>
- </condition>
+ <definition xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_indexed_wal_edit_codec">
+ <type>hbase-site</type>
+ <set key="hbase.regionserver.wal.codec"
+ value="org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec"
+ if-type="hbase-env" if-key="phoenix_sql_enabled" if-value="true"/>
</definition>
<definition xsi:type="configure"
@@ -1007,8 +977,7 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<changes>
- <definition xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations">
- <summary>Updating oozie-site to remove redundant configurations</summary>
+ <definition xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations" summary="Updating oozie-site to remove redundant configurations">
<type>oozie-site</type>
<transfer operation="delete" delete-key="*" preserve-edits="true">
<keep-key>oozie.base.url</keep-key>
@@ -1127,5 +1096,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index 7d80a63..00603a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -16,8 +16,7 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="HBASE">
<component name="HBASE_MASTER">
@@ -100,16 +99,12 @@
<!-- Add these configs if the cluster is Kerberized.
Will only be written to the local file system if Atlas is present. -->
<definition xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs">
- <condition type="cluster-env" key="security_enabled" value="true">
- <type>sqoop-atlas-application.properties</type>
- <key>atlas.jaas.KafkaClient.option.useTicketCache</key>
- <value>true</value>
- </condition>
- <condition type="cluster-env" key="security_enabled" value="true">
- <type>sqoop-atlas-application.properties</type>
- <key>atlas.jaas.KafkaClient.option.renewTicket</key>
- <value>true</value>
- </condition>
+ <type>sqoop-atlas-application.properties</type>
+ <set key="atlas.jaas.KafkaClient.option.useTicketCache" value="true"
+ if-type="cluster-env" if-key="security_enabled" if-value="true"/>
+
+ <set key="atlas.jaas.KafkaClient.option.renewTicket" value="true"
+ if-type="cluster-env" if-key="security_enabled" if-value="true"/>
</definition>
</changes>
</component>
@@ -228,11 +223,9 @@
</definition>
<definition xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag">
- <condition type="ranger-env" key="is_solrCloud_enabled" value="true">
- <type>ranger-env</type>
- <key>is_external_solrCloud_enabled</key>
- <value>true</value>
- </condition>
+ <type>ranger-env</type>
+ <set key="is_external_solrCloud_enabled" value="true"
+ if-type="ranger-env" if-key="is_solrCloud_enabled" if-value="true"/>
</definition>
</changes>
@@ -488,5 +481,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index ecf324c..c791204 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -16,10 +16,8 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
-
<service name="TEZ">
<component name="TEZ_CLIENT">
<changes>
@@ -48,16 +46,12 @@
<!-- Add these configs if the cluster is Kerberized.
Will only be written to the local file system if Atlas is present. -->
<definition xsi:type="configure" id="hdp_2_5_0_0_add_sqoop_atlas_security_configs">
- <condition type="cluster-env" key="security_enabled" value="true">
- <type>sqoop-atlas-application.properties</type>
- <key>atlas.jaas.KafkaClient.option.useTicketCache</key>
- <value>true</value>
- </condition>
- <condition type="cluster-env" key="security_enabled" value="true">
- <type>sqoop-atlas-application.properties</type>
- <key>atlas.jaas.KafkaClient.option.renewTicket</key>
- <value>true</value>
- </condition>
+ <type>sqoop-atlas-application.properties</type>
+ <set key="atlas.jaas.KafkaClient.option.useTicketCache" value="true"
+ if-type="cluster-env" if-key="security_enabled" if-value="true"/>
+
+ <set key="atlas.jaas.KafkaClient.option.renewTicket" value="true"
+ if-type="cluster-env" if-key="security_enabled" if-value="true"/>
</definition>
</changes>
</component>
@@ -135,11 +129,9 @@
</definition>
<definition xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag">
- <condition type="ranger-env" key="is_solrCloud_enabled" value="true">
- <type>ranger-env</type>
- <key>is_external_solrCloud_enabled</key>
- <value>true</value>
- </condition>
+ <type>ranger-env</type>
+ <set key="is_external_solrCloud_enabled" value="true"
+ if-type="ranger-env" if-key="is_solrCloud_enabled" if-value="true"/>
</definition>
</changes>
@@ -360,5 +352,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 59e4ec5..87ede63 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -16,8 +16,7 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="STORM">
<component name="NIMBUS">
@@ -46,5 +45,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/main/resources/upgrade-config.xsd
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade-config.xsd b/ambari-server/src/main/resources/upgrade-config.xsd
new file mode 100644
index 0000000..e274451
--- /dev/null
+++ b/ambari-server/src/main/resources/upgrade-config.xsd
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" version="1.1">
+ <xs:annotation>
+ <xs:documentation>
+ This document describes the schema for Upgrade Pack configuration changes.
+ </xs:documentation>
+ </xs:annotation>
+
+ <xs:simpleType name="transfer-operation-type">
+ <xs:restriction base="xs:string">
+ <xs:enumeration value="delete"/>
+ <xs:enumeration value="move"/>
+ <xs:enumeration value="copy"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="set-if-key-state-type">
+ <xs:restriction base="xs:string">
+ <xs:enumeration value="present"/>
+ <xs:enumeration value="absent"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="transfer-operation-coerce-type">
+ <xs:restriction base="xs:string">
+ <xs:enumeration value="yaml-array"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:complexType name="configure">
+ <xs:sequence>
+ <xs:element name="type" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="transfer" minOccurs="0" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="keep-key" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ <xs:attribute name="operation" use="required" type="transfer-operation-type"/>
+ <xs:attribute name="from-type" use="optional" type="xs:string"/>
+ <xs:attribute name="from-key" use="optional" type="xs:string"/>
+ <xs:attribute name="to-key" use="optional" type="xs:string"/>
+ <xs:attribute name="delete-key" use="optional" type="xs:string"/>
+ <xs:attribute name="preserve-edits" use="optional" type="xs:boolean"/>
+ <xs:attribute name="default-value" use="optional" type="xs:string"/>
+ <xs:attribute name="coerce-to" use="optional" type="transfer-operation-coerce-type"/>
+ <xs:attribute name="if-key" use="optional" type="xs:string"/>
+ <xs:attribute name="if-type" use="optional" type="xs:string"/>
+ <xs:attribute name="if-value" use="optional" type="xs:string"/>
+ <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/>
+ <xs:attribute name="mask" use="optional" type="xs:boolean"/>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="set" minOccurs="0" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:attribute name="key" use="required" type="xs:string"/>
+ <xs:attribute name="value" use="required" type="xs:string"/>
+ <xs:attribute name="if-key" use="optional" type="xs:string"/>
+ <xs:attribute name="if-type" use="optional" type="xs:string"/>
+ <xs:attribute name="if-value" use="optional" type="xs:string"/>
+ <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/>
+ <xs:attribute name="mask" use="optional" type="xs:boolean"/>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="replace" minOccurs="0" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:attribute name="key" use="required" type="xs:string"/>
+ <xs:attribute name="find" use="required" type="xs:string"/>
+ <xs:attribute name="replace-with" use="required" type="xs:string"/>
+ <xs:attribute name="if-key" use="optional" type="xs:string"/>
+ <xs:attribute name="if-type" use="optional" type="xs:string"/>
+ <xs:attribute name="if-value" use="optional" type="xs:string"/>
+ <xs:attribute name="if-key-state" use="optional" type="set-if-key-state-type"/>
+ <xs:attribute name="mask" use="optional" type="xs:boolean"/>
+ </xs:complexType>
+ </xs:element>
+ </xs:choice>
+ </xs:sequence>
+ <xs:attribute name="id" use="required" type="xs:string"/>
+ <xs:attribute name="summary" use="optional" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="changes-type">
+ <xs:sequence>
+ <xs:element name="definition" type="configure" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="component-type">
+ <xs:sequence>
+ <xs:element name="changes" type="changes-type" minOccurs="1" maxOccurs="1">
+ <xs:unique name="unique-by-definition-id">
+ <xs:annotation>
+ <xs:documentation>Ensures that the element "changes" does not have duplicate definitions</xs:documentation>
+ </xs:annotation>
+ <xs:selector xpath="definition"/>
+ <xs:field xpath="@id"/>
+ </xs:unique>
+ </xs:element>
+ </xs:sequence>
+ <xs:attribute name="name" use="required" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="service-type">
+ <xs:sequence>
+ <xs:element name="component" type="component-type" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ <xs:attribute name="name" use="required" type="xs:string"/>
+ </xs:complexType>
+
+ <xs:complexType name="services-type">
+ <xs:sequence>
+ <xs:element name="service" type="service-type" minOccurs="1" maxOccurs="unbounded">
+ <xs:unique name="unique-by-component">
+ <xs:annotation>
+ <xs:documentation>Ensures that the element "service" does not have duplicate components</xs:documentation>
+ </xs:annotation>
+ <xs:selector xpath="component" />
+ <xs:field xpath="@name" />
+ </xs:unique>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:element name="upgrade-config-changes">
+ <xs:annotation>
+ <xs:documentation>
+ This is the root element of an the configuration changes for an Upgrade Pack.
+ </xs:documentation>
+ </xs:annotation>
+
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="services" type="services-type" minOccurs="1">
+ <xs:unique name="unique-by-service">
+ <xs:annotation>
+ <xs:documentation>Ensures that the element "services" does not have duplicate services</xs:documentation>
+ </xs:annotation>
+ <xs:selector xpath="service" />
+ <xs:field xpath="@name" />
+ </xs:unique>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+</xs:schema>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index a1d4c4b..d644a09 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -53,6 +53,7 @@ import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition;
+import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue;
import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.apache.ambari.server.state.stack.upgrade.ExecuteTask;
@@ -658,7 +659,7 @@ public class UpgradeHelperTest {
// grab the configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
assertEquals("HIVE", hiveGroup.name);
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0);
// now change the thrift port to http to have the 2nd condition invoked
Map<String, String> hiveConfigs = new HashMap<String, String>();
@@ -732,7 +733,8 @@ public class UpgradeHelperTest {
assertEquals("HIVE", hiveGroup.name);
//Condition is met
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(
+ 0).getTasks().get(0);
Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
@@ -841,7 +843,7 @@ public class UpgradeHelperTest {
assertEquals("HIVE", hiveGroup.name);
//Condition is not met, so no config operations should be present in the configureTask...
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(4).getTasks().get(0).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0);
Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
@@ -853,7 +855,6 @@ public class UpgradeHelperTest {
String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS);
String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS);
- System.out.println(" testConfigTaskConditionSkip >> transferJson"+transferJson);
String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS);
assertNotNull(configurationJson);
@@ -876,6 +877,12 @@ public class UpgradeHelperTest {
assertTrue(transfers.isEmpty());
}
+ /**
+ * Tests that {@link ConfigurationKeyValue} pairs on a {@link ConfigureTask}
+ * are correctly returned based on the if-conditions.
+ *
+ * @throws Exception
+ */
@Test
public void testConfigureTask() throws Exception {
Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
@@ -894,30 +901,19 @@ public class UpgradeHelperTest {
assertEquals(7, groups.size());
- // grab the configure task out of Hive
+ // grab the first configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
assertEquals("HIVE", hiveGroup.name);
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(
- 0).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0);
Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site");
- String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS);
- assertNotNull(configurationJson);
-
- List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(configurationJson,
- new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() {
- }.getType());
-
- assertEquals("hive.server2.thrift.port", keyValuePairs.get(0).key);
- assertEquals("10010", keyValuePairs.get(0).value);
-
- // now change the thrift port to http to have the 2nd condition invoked
+ // now set the property in the if-check in the set element so that we have a match
Map<String, String> hiveConfigs = new HashMap<String, String>();
- hiveConfigs.put("hive.server2.transport.mode", "http");
- hiveConfigs.put("hive.server2.thrift.port", "10001");
+ hiveConfigs.put("fooKey", "THIS-BETTER-CHANGE");
+ hiveConfigs.put("ifFooKey", "ifFooValue");
ConfigurationRequest configurationRequest = new ConfigurationRequest();
configurationRequest.setClusterName(cluster.getClusterName());
configurationRequest.setType("hive-site");
@@ -935,20 +931,22 @@ public class UpgradeHelperTest {
}
}, null);
- // the configure task should now return different properties
+ // the configure task should now return different properties to set based on
+ // the if-condition checks
configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
assertEquals( configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site");
- configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS);
+ String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS);
assertNotNull(configurationJson);
- keyValuePairs = m_gson.fromJson(configurationJson,
+ List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue> keyValuePairs = m_gson.fromJson(
+ configurationJson,
new TypeToken<List<ConfigUpgradeChangeDefinition.ConfigurationKeyValue>>() {
}.getType());
- assertEquals("hive.server2.http.port", keyValuePairs.get(0).key);
- assertEquals("10011", keyValuePairs.get(0).value);
+ assertEquals("fooKey", keyValuePairs.get(0).key);
+ assertEquals("fooValue", keyValuePairs.get(0).value);
}
@Test
@@ -970,7 +968,7 @@ public class UpgradeHelperTest {
// grab the configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
assertEquals("HIVE", hiveGroup.name);
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0);
Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java
index e764781..ee1b05e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.state.stack;
+import java.io.File;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -26,6 +28,7 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.internal.UpgradeResourceProvider.ConfigurationPackBuilder;
import org.apache.ambari.server.orm.GuiceJpaInitializer;
import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.stack.ModuleFileUnmarshaller;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
@@ -36,6 +39,9 @@ import org.apache.ambari.server.state.stack.upgrade.Direction;
import org.apache.ambari.server.state.stack.upgrade.Grouping;
import org.apache.ambari.server.state.stack.upgrade.Task;
import org.apache.ambari.server.state.stack.upgrade.Task.Type;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +57,8 @@ import junit.framework.Assert;
/**
* Tests that for every upgrade pack found, that all referenced configuration
- * IDs exist in the {@code config-upgrade.xml} which will be used/created.
+ * IDs exist in the {@code config-upgrade.xml} which will be used/created. Also
+ * ensures that every XML file is valid against its XSD.
*/
@Category({ category.StackUpgradeTest.class})
public class ConfigUpgradeValidityTest {
@@ -167,6 +174,65 @@ public class ConfigUpgradeValidityTest {
Assert.assertTrue(validatedConfigCount > 100);
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testValidateConfigUpgradePacks() throws Exception {
+ IOFileFilter filter = new IOFileFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return false;
+ }
+
+ @Override
+ public boolean accept(File file) {
+ // file has the folder named 'upgrades', ends with '.xml' and is NOT
+ // 'config-upgrade.xml'
+ if (file.getAbsolutePath().contains("upgrades")
+ && file.getAbsolutePath().endsWith("config-upgrade.xml")) {
+ return true;
+ }
+
+ return false;
+ }
+ };
+
+ List<File> files = new ArrayList<>();
+
+ files.addAll(FileUtils.listFiles(new File("src/main/resources/stacks"), filter,
+ FileFilterUtils.directoryFileFilter()));
+
+ files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks"), filter,
+ FileFilterUtils.directoryFileFilter()));
+
+ files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks_with_upgrade_cycle"),
+ filter, FileFilterUtils.directoryFileFilter()));
+
+ ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller();
+
+ int filesTestedCount = 0;
+ for (File file : files) {
+ String fileContent = FileUtils.readFileToString(file, "UTF-8");
+
+ // these things must be in upgrade packs for them to work anyway
+ if (fileContent.contains("<upgrade-config-changes")
+ && fileContent.contains("xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"")) {
+ if (!fileContent.contains("xsi:noNamespaceSchemaLocation=\"upgrade-config.xsd\"")) {
+ String msg = String.format(
+ "File %s appears to be a config upgrade pack, but does not define 'upgrade-config.xsd' as its schema",
+ file.getAbsolutePath());
+ Assert.fail(msg);
+ } else {
+ filesTestedCount++;
+ unmarshaller.unmarshal(ConfigUpgradePack.class, file, true);
+ }
+ }
+ }
+
+ Assert.assertTrue(
+ "This test didn't appear to do any work which could indicate that it failed to find files to validate",
+ filesTestedCount > 5);
+ }
+
/**
* Asserts that an ID exists in a {@link ConfigUpgradePack}, throwing an
* informative message if it does not.
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
index 44f9e02..307f4d4 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml
@@ -16,13 +16,14 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
<changes>
<definition xsi:type="configure" id="hdp_2_1_1_zk_post_upgrade">
+ <type>zookeeper-newconfig</type>
+ <set key="fooKey" value="fooValue"/>
</definition>
<definition xsi:type="configure" id="hdp_2_1_1_zookeeper_new_config_type">
@@ -48,8 +49,7 @@
<definition xsi:type="configure" id="hdp_2_1_1_nn_test">
<type>hdfs-site</type>
- <key>myproperty</key>
- <value>mynewvalue</value>
+ <set key="myproperty" value="mynewvalue"/>
</definition>
<definition xsi:type="configure" id="hdp_2_1_1_hdfs_new_config_type">
<type>hdfs-newconfig</type>
@@ -83,35 +83,9 @@
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
- <definition xsi:type="configure" id="hdp_2_1_1_set_transport_mode">
- <condition type="hive-site" key="hive.server2.transport.mode" value="binary">
- <type>hive-site</type>
- <key>hive.server2.thrift.port</key>
- <value>10010</value>
- </condition>
- <condition type="hive-site" key="hive.server2.transport.mode" value="http">
- <type>hive-site</type>
- <key>hive.server2.http.port</key>
- <value>10011</value>
- </condition>
- </definition>
-
- <definition xsi:type="configure" id="hdp_2_1_1_test_properties">
- <condition type="hive-site" key="hive.server2.transport.mode" value="binary">
- <type>hive-site</type>
- <key>hive.server2.thrift.port</key>
- <value>10010</value>
- </condition>
- <condition type="hive-site" key="hive.server2.transport.mode" value="http">
- <type>hive-site</type>
- <key>hive.server2.http.port</key>
- <value>10011</value>
- </condition>
- </definition>
-
<definition xsi:type="configure" id="hdp_2_1_1_hive_server_foo">
<type>hive-site</type>
- <set key="fooKey" value="fooValue"/>
+ <set key="fooKey" value="fooValue" if-type="hive-site" if-key="ifFooKey" if-value="ifFooValue"/>
<set key="fooKey2" value="fooValue2"/>
<set key="fooKey3" value="fooValue3"/>
<transfer operation="copy" from-key="copy-key" to-key="copy-key-to" />
@@ -181,7 +155,7 @@
<!-- set -->
<set key="setKeyOne" value="1" if-key="hive.server2.transport.mode" if-type="" if-value="skip"/>
<set key="setKeyTwo" value="2" if-key="" if-type="hive-site" if-key-state="absent"/>
- <set key="setKeyThree" value="3" if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/>
+ <set key="setKeyThree" value="3" if-key="" if-type="hive-site" if-key-state="present"/>
<set key="setKeyThree" value="3" if-key="foo.bar" if-type="hive-site" />
<!-- transfer operation Copy -->
@@ -192,7 +166,7 @@
<transfer operation="copy" from-type="hive-site" from-key="copy-key-three" to-key="copy-to-key-four" default-value="1"
if-key="hive.server2.transport.mode" if-type="hive-site" />
<transfer operation="copy" from-type="hive-site" from-key="copy-key-four" to-key="copy-to-key-four" default-value="1"
- if-key="hive.server2.transport.mode" if-type="hive-site" if-key-state="abcd"/>
+ if-key="hive.server2.transport.mode" if-type="" if-key-state="present"/>
<!-- transfer operation move -->
<transfer operation="move" from-type="hive-site" from-key="move-key-one" to-key="move-to-key-four" default-value="1"
@@ -200,14 +174,14 @@
<transfer operation="move" from-type="hive-site" from-key="move-key-two" to-key="move-to-key-two" default-value="1"
if-key="" if-type="hive-site" if-key-state="absent"/>
<transfer operation="move" from-type="hive-site" from-key="move-key-three" to-key="move-to-key-three" default-value="1"
- if-key="hive.server2.transport.mode" if-type="hive-site" if-key-state="abcd"/>
+ if-key="hive.server2.transport.mode" if-type="" if-key-state="present"/>
<transfer operation="move" from-type="hive-site" from-key="move-key-four" to-key="move-to-key-four" default-value="1"
if-key="hive.server2.transport.mode" if-type="hive-site"/>
<!-- transfer operation delete -->
<transfer operation="delete" delete-key="delete-key-one" if-key="hive.server2.transport.mode" if-type="" if-key-state="absent"/>
<transfer operation="delete" delete-key="delete-key-two" if-key="" if-type="hive-site" if-key-state="absent"/>
- <transfer operation="delete" delete-key="delete-key-three" if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/>
+ <transfer operation="delete" delete-key="delete-key-three" if-key="foo.bar" if-type="" if-key-state="present"/>
<transfer operation="delete" delete-key="delete-key-four" if-key="hive.server2.transport.mode" if-type="hive-site" />
<!-- replacement -->
@@ -216,13 +190,17 @@
<replace key="replace-key-two" find="efg" replace-with="efg-replaced"
if-key="" if-type="hive-site" if-key-state="absent"/>
<replace key="replace-key-three" find="ijk" replace-with="ijk-replaced"
- if-key="foo.bar" if-type="hive-site" if-key-state="abcd"/>
+ if-key="foo.bar" if-type="" if-key-state="present"/>
<replace key="replace-key-three" find="ijk" replace-with="ijk-replaced"
if-key="foo.bar" if-type="hive-site"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_1_1_no_conditions_met">
+ <type>hive-site</type>
+ <set key="fooKey" value="fooValue" if-type="hive-site" if-key="ifFooKey" if-value="ifFooValue"/>
+ </definition>
</changes>
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
index 88bb73e..5e02b15 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
@@ -217,10 +217,10 @@
<summary>HiveServer Port Availability</summary>
<message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
</task>
- <task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/>
<task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/>
<task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/>
<task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/>
+ <task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/>
</pre-upgrade>
<upgrade />
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml
index 90d64b4..dc7465e 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml
@@ -16,13 +16,14 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
<changes>
<definition xsi:type="configure" id="hdp_2_2_0_zk_post_upgrade">
+ <type>zookeeper-newconfig</type>
+ <set key="fooKey" value="fooValue"/>
</definition>
</changes>
</component>
@@ -63,19 +64,6 @@
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
- <definition xsi:type="configure" id="hdp_2_2_0_set_transport_mode">
- <condition type="hive-site" key="hive.server2.transport.mode" value="binary">
- <type>hive-site</type>
- <key>hive.server2.thrift.port</key>
- <value>10010</value>
- </condition>
- <condition type="hive-site" key="hive.server2.transport.mode" value="http">
- <type>hive-site</type>
- <key>hive.server2.http.port</key>
- <value>10011</value>
- </condition>
- </definition>
-
<definition xsi:type="configure" id="hdp_2_2_0_hive_server_foo">
<type>hive-site</type>
<set key="fooKey" value="fooValue"/>
@@ -97,5 +85,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7b30be6d/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml
index 90d64b4..d40faba 100644
--- a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml
+++ b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml
@@ -16,13 +16,14 @@
limitations under the License.
-->
-<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-
+<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-config.xsd">
<services>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
<changes>
<definition xsi:type="configure" id="hdp_2_2_0_zk_post_upgrade">
+ <type>zookeeper-newconfig</type>
+ <set key="fooKey" value="fooValue"/>
</definition>
</changes>
</component>
@@ -63,19 +64,6 @@
<service name="HIVE">
<component name="HIVE_SERVER">
<changes>
- <definition xsi:type="configure" id="hdp_2_2_0_set_transport_mode">
- <condition type="hive-site" key="hive.server2.transport.mode" value="binary">
- <type>hive-site</type>
- <key>hive.server2.thrift.port</key>
- <value>10010</value>
- </condition>
- <condition type="hive-site" key="hive.server2.transport.mode" value="http">
- <type>hive-site</type>
- <key>hive.server2.http.port</key>
- <value>10011</value>
- </condition>
- </definition>
-
<definition xsi:type="configure" id="hdp_2_2_0_hive_server_foo">
<type>hive-site</type>
<set key="fooKey" value="fooValue"/>
@@ -97,5 +85,4 @@
</component>
</service>
</services>
-
</upgrade-config-changes>
[03/16] ambari git commit: AMBARI-18704. Add code to improve
debugging of ambari-agent related problems. (aonishuk)
Posted by nc...@apache.org.
AMBARI-18704. Add code to improve debugging of ambari-agent related problems. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa588ca8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa588ca8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa588ca8
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: aa588ca8667f5f67b6be2251a6dad37230172fb4
Parents: 7b30be6
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Oct 26 20:08:46 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Wed Oct 26 20:08:46 2016 +0300
----------------------------------------------------------------------
.../python/ambari_agent/HeartbeatHandlers.py | 13 ++++--------
.../python/ambari_agent/RemoteDebugUtils.py | 21 +++++++++++++++++++-
.../ambari_agent/StatusCommandsExecutor.py | 12 ++++++++---
3 files changed, 33 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa588ca8/ambari-agent/src/main/python/ambari_agent/HeartbeatHandlers.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HeartbeatHandlers.py b/ambari-agent/src/main/python/ambari_agent/HeartbeatHandlers.py
index 4a3d372..836ab07 100644
--- a/ambari-agent/src/main/python/ambari_agent/HeartbeatHandlers.py
+++ b/ambari-agent/src/main/python/ambari_agent/HeartbeatHandlers.py
@@ -26,9 +26,10 @@ import signal
import threading
import traceback
from ambari_commons.os_family_impl import OsFamilyImpl
-from RemoteDebugUtils import remote_debug
import sys
+from ambari_agent.RemoteDebugUtils import bind_debug_signal_handlers
+
logger = logging.getLogger()
_handler = None
@@ -128,14 +129,8 @@ def bind_signal_handlers(agentPid):
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
- signal.signal(signal.SIGUSR2, remote_debug) # Interrupt running process, and provide a python prompt for it
- try:
- import faulthandler # This is not default module, has to be installed separately
- faulthandler.enable(file=sys.stderr, all_threads=True)
- faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True, chain=False)
- sys.stderr.write("Registered faulthandler\n")
- except ImportError:
- pass # Module is not included into python distribution
+
+ bind_debug_signal_handlers()
_handler = HeartbeatStopHandlersLinux()
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa588ca8/ambari-agent/src/main/python/ambari_agent/RemoteDebugUtils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/RemoteDebugUtils.py b/ambari-agent/src/main/python/ambari_agent/RemoteDebugUtils.py
index f2a462b..ae997ac 100644
--- a/ambari-agent/src/main/python/ambari_agent/RemoteDebugUtils.py
+++ b/ambari-agent/src/main/python/ambari_agent/RemoteDebugUtils.py
@@ -21,7 +21,26 @@ limitations under the License.
try: import readline # For readline input support
except: pass
-import sys, os, traceback, codeop, cStringIO, cPickle, tempfile
+import sys, signal, os, traceback, codeop, cStringIO, cPickle, tempfile
+
+def bind_debug_signal_handlers():
+ signal.signal(signal.SIGUSR1, print_threads_stack_traces) # prints process threads current stack trace to the err stream. (can be found in ambari-agent.out)
+ signal.signal(signal.SIGUSR2, remote_debug) # provide a read-only python shell, which represent the process state at time of signal arrival.
+
+def print_threads_stack_traces(sig, frame):
+ print >> sys.stderr, "\n*** STACKTRACE - START ***\n"
+ code = []
+ for threadId, stack in sys._current_frames().items():
+ code.append("\n# ThreadID: %s" % threadId)
+ for filename, lineno, name, line in traceback.extract_stack(stack):
+ code.append('File: "%s", line %d, in %s' % (filename,
+ lineno, name))
+ if line:
+ code.append(" %s" % (line.strip()))
+
+ for line in code:
+ print >> sys.stderr, line
+ print >> sys.stderr, "\n*** STACKTRACE - END ***\n"
def pipename(pid):
"""Return name of pipe to use"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/aa588ca8/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py b/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
index 8959640..20acee4 100644
--- a/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
+++ b/ambari-agent/src/main/python/ambari_agent/StatusCommandsExecutor.py
@@ -22,7 +22,8 @@ import signal
import threading
import logging
import multiprocessing
-from PythonReflectiveExecutor import PythonReflectiveExecutor
+from ambari_agent.PythonReflectiveExecutor import PythonReflectiveExecutor
+from ambari_agent.RemoteDebugUtils import bind_debug_signal_handlers
logger = logging.getLogger(__name__)
@@ -43,8 +44,10 @@ class StatusCommandsExecutor(multiprocessing.Process):
def run(self):
try:
+ bind_debug_signal_handlers()
while True:
command = self.actionQueue.statusCommandQueue.get(True) # blocks until status status command appears
+ logger.info("Running status command for {0}".format(command['componentName'])) # TODO: change to logger.debug once fixed
timeout_timer = threading.Timer( self.status_command_timeout, self.respawn, [command])
timeout_timer.start()
@@ -52,6 +55,7 @@ class StatusCommandsExecutor(multiprocessing.Process):
self.process_status_command(command)
timeout_timer.cancel()
+ logger.info("Completed status command for {0}".format(command['componentName'])) # TODO: change to logger.debug once fixed
except:
logger.exception("StatusCommandsExecutor process failed with exception:")
raise
@@ -67,8 +71,10 @@ class StatusCommandsExecutor(multiprocessing.Process):
def respawn(self, command):
try:
- # Force context to reset to normal. By context we mean sys.path, imports, etc. They are set by specific status command, and are not relevant to ambari-agent.
- PythonReflectiveExecutor.last_context.revert()
+ if hasattr(PythonReflectiveExecutor, "last_context"):
+ # Force context to reset to normal. By context we mean sys.path, imports, etc. They are set by specific status command, and are not relevant to ambari-agent.
+ PythonReflectiveExecutor.last_context.revert()
+
logger.warn("Command {0} for {1} is running for more than {2} seconds. Terminating it due to timeout.".format(command['commandType'], command['componentName'], self.status_command_timeout))
self.hasTimeoutedEvent.set()
[14/16] ambari git commit: AMBARI-18717. Python UT fail on trunk
(aonishuk)
Posted by nc...@apache.org.
AMBARI-18717. Python UT fail on trunk (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7e4f6209
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7e4f6209
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7e4f6209
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 7e4f6209649e8f1ccb86e04ce1e2f54225e5375c
Parents: 6cc4431
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Oct 27 16:36:11 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 27 16:36:11 2016 +0300
----------------------------------------------------------------------
.../src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e4f6209/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
index 9ec3b51..24cf271 100644
--- a/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
+++ b/ambari-server/src/test/python/stacks/2.5/ZEPPELIN/test_zeppelin_master.py
@@ -296,6 +296,9 @@ class TestZeppelinMaster(RMFTestCase):
self.assertResourceCalled('Execute', ('chown', '-R', u'zeppelin:zeppelin', '/etc/zeppelin'),
sudo = True,
)
+ self.assertResourceCalled('Execute', ('chown', '-R', 'zeppelin:zeppelin', '/usr/hdp/current/zeppelin-server/notebook'),
+ sudo = True,
+ )
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/zeppelin.server.kerberos.keytab zeppelin@EXAMPLE.COM; ',
user = 'zeppelin',
)
[16/16] ambari git commit: Merge branch 'trunk' into
branch-feature-AMBARI-18634
Posted by nc...@apache.org.
Merge branch 'trunk' into branch-feature-AMBARI-18634
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6088e4f1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6088e4f1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6088e4f1
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 6088e4f1c96c27ada12e1645db828a234efbe876
Parents: ba526d9 91b7d22
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Oct 27 11:20:13 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu Oct 27 11:20:13 2016 -0400
----------------------------------------------------------------------
.../python/ambari_agent/HeartbeatHandlers.py | 13 +-
.../python/ambari_agent/RemoteDebugUtils.py | 21 ++-
.../ambari_agent/StatusCommandsExecutor.py | 12 +-
.../libraries/functions/constants.py | 1 +
.../src/main/python/core/host_info.py | 12 +-
.../internal/UserPrivilegeResourceProvider.java | 116 ++++++++++++-
.../ambari/server/events/AmbariEvent.java | 5 +
.../server/events/StackUpgradeFinishEvent.java | 41 +++++
.../upgrade/StackUpgradeFinishListener.java | 83 ++++++++++
.../listeners/upgrade/StackVersionListener.java | 53 ++++--
.../publishers/VersionEventPublisher.java | 9 +-
.../upgrades/FinalizeUpgradeAction.java | 8 +
.../ambari/server/state/ServiceComponent.java | 6 +
.../server/state/ServiceComponentImpl.java | 41 ++---
.../upgrade/ConfigUpgradeChangeDefinition.java | 75 +--------
.../state/stack/upgrade/ConfigureTask.java | 49 +-----
.../ATLAS/0.1.0.2.3/role_command_order.json | 7 +
.../ATLAS/0.7.0.2.5/role_command_order.json | 7 +
.../HAWQ/2.0.0/role_command_order.json | 11 ++
.../2.1.0.2.0/package/scripts/zkfc_slave.py | 20 +++
.../PXF/3.0.0/configuration/pxf-profiles.xml | 14 ++
.../PXF/3.0.0/role_command_order.json | 7 +
.../0.6.0.2.5/package/scripts/master.py | 2 +
.../HDP/2.0.6/properties/stack_features.json | 5 +
.../stacks/HDP/2.2/upgrades/config-upgrade.xml | 110 +++++--------
.../stacks/HDP/2.3/role_command_order.json | 10 --
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 28 ++--
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 29 ++--
.../stacks/HDP/2.5/role_command_order.json | 3 +-
.../stacks/HDP/2.5/services/HDFS/metainfo.xml | 6 +
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 4 +-
.../src/main/resources/upgrade-config.xsd | 163 +++++++++++++++++++
.../UserPrivilegeResourceProviderTest.java | 2 +
.../upgrade/StackUpgradeFinishListenerTest.java | 108 ++++++++++++
.../upgrade/StackVersionListenerTest.java | 47 +++++-
.../ambari/server/state/UpgradeHelperTest.java | 50 +++---
.../state/stack/ConfigUpgradeValidityTest.java | 68 +++++++-
.../stacks/2.5/ZEPPELIN/test_zeppelin_master.py | 3 +
.../HDP/2.1.1/upgrades/config-upgrade.xml | 52 ++----
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 2 +-
.../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +--
.../HDP/2.2.0/upgrades/config-upgrade.xml | 19 +--
.../controllers/main/host/combo_search_box.js | 2 +-
.../app/routes/manage_journalnode_routes.js | 3 +-
ambari-web/app/styles/application.less | 5 +
.../highAvailability/journalNode/wizard.hbs | 34 ++--
.../app/templates/main/service/menu_item.hbs | 4 +-
.../views/common/configs/config_history_flow.js | 2 +-
48 files changed, 965 insertions(+), 426 deletions(-)
----------------------------------------------------------------------
[13/16] ambari git commit: AMBARI-18645. Move ATLAS role command
order to common-services/ATLAS (aonishuk)
Posted by nc...@apache.org.
AMBARI-18645. Move ATLAS role command order to common-services/ATLAS (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6cc4431b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6cc4431b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6cc4431b
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 6cc4431b8258fab1c5fbf614734e916b6dee2bad
Parents: 9c6809f
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Oct 27 14:25:25 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 27 14:25:25 2016 +0300
----------------------------------------------------------------------
.../common-services/ATLAS/0.1.0.2.3/role_command_order.json | 7 +++++++
.../common-services/ATLAS/0.7.0.2.5/role_command_order.json | 7 +++++++
.../src/main/resources/stacks/HDP/2.3/role_command_order.json | 2 --
.../src/main/resources/stacks/HDP/2.5/role_command_order.json | 3 +--
4 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc4431b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/role_command_order.json b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/role_command_order.json
new file mode 100644
index 0000000..4d66dfc
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for ATLAS",
+ "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
+ "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "INFRA_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc4431b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/role_command_order.json b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/role_command_order.json
new file mode 100644
index 0000000..372cd21
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/role_command_order.json
@@ -0,0 +1,7 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for ATLAS",
+ "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
+ "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "LOGSEARCH_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc4431b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 2c841ca..7626630 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -8,8 +8,6 @@
"RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
"RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
"PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
- "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
- "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "INFRA_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
"SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],
"RESOURCEMANAGER-STOP" : ["SPARK_THRIFTSERVER-STOP"],
"KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],
http://git-wip-us.apache.org/repos/asf/ambari/blob/6cc4431b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
index f2616b0..5a9825d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json
@@ -12,8 +12,7 @@
"RANGER_ADMIN-START": ["ZOOKEEPER_SERVER-START", "INFRA_SOLR-START"],
"LIVY_SERVER-START" : ["NAMENODE-START", "DATANODE-START", "APP_TIMELINE_SERVER-START"],
"SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "LIVY_SERVER-START"],
- "SPARK2_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK2_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"],
- "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "LOGSEARCH_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"]
+ "SPARK2_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK2_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"]
},
"_comment" : "Dependencies that are used when GLUSTERFS is not present in cluster",
"optional_no_glusterfs": {
[15/16] ambari git commit: AMBARI-18718. Status Icons taking up a new
line in Mozilla Firefox (Windows) (akovalenko)
Posted by nc...@apache.org.
AMBARI-18718. Status Icons taking up a new line in Mozilla Firefox (Windows) (akovalenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/91b7d221
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/91b7d221
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/91b7d221
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 91b7d221256943f0b000b31cab407557f15bc91c
Parents: 7e4f620
Author: Aleksandr Kovalenko <ak...@hortonworks.com>
Authored: Thu Oct 27 17:10:27 2016 +0300
Committer: Aleksandr Kovalenko <ak...@hortonworks.com>
Committed: Thu Oct 27 17:10:27 2016 +0300
----------------------------------------------------------------------
ambari-web/app/templates/main/service/menu_item.hbs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/91b7d221/ambari-web/app/templates/main/service/menu_item.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/service/menu_item.hbs b/ambari-web/app/templates/main/service/menu_item.hbs
index 2da1555..78cce61 100644
--- a/ambari-web/app/templates/main/service/menu_item.hbs
+++ b/ambari-web/app/templates/main/service/menu_item.hbs
@@ -17,8 +17,6 @@
}}
<a class="services-menu-blocks" {{bindAttr href="view.link"}}>
- {{view App.MainDashboardServiceHealthView class="service-health" serviceBinding="view.content"}}
- {{unbound view.content.displayName}}
{{#if view.alertsCount}}
<span {{bindAttr class=":label :alerts-count view.hasCriticalAlerts:alerts-crit-count:alerts-warn-count :pull-right"}}>
{{view.alertsCount}}
@@ -27,4 +25,6 @@
<span class="pull-right">
<i rel="tooltip" {{action goToConfigs target="view"}} {{bindAttr class=":glyphicon :glyphicon-refresh :restart-required-service view.content.isRestartRequired::hidden" data-original-title="view.restartRequiredMessage"}}></i>
</span>
+ {{view App.MainDashboardServiceHealthView class="service-health" serviceBinding="view.content"}}
+ {{unbound view.content.displayName}}
</a>
[11/16] ambari git commit: AMBARI-18677. Zeppelin install not
correctly setting user and group for notebook directory
Posted by nc...@apache.org.
AMBARI-18677. Zeppelin install not correctly setting user and group for notebook directory
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b09b63b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b09b63b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b09b63b
Branch: refs/heads/branch-feature-AMBARI-18634
Commit: 4b09b63bfd2df05d901f40f2496d315f95f5df9e
Parents: d52154e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Oct 26 18:10:11 2016 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Oct 26 18:11:09 2016 -0700
----------------------------------------------------------------------
.../common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b09b63b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 8eb36c8..4583d81 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -161,6 +161,8 @@ class Master(Script):
Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"), "/etc/zeppelin"),
sudo=True)
+ Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"),
+ os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
if params.security_enabled:
zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ")