You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2017/10/11 08:38:45 UTC
[7/7] ambari git commit: AMBARI-22190. After merging trunk to
branch-3.0-perf some parts of code are missing. (mpapirkovskyy)
AMBARI-22190. After merging trunk to branch-3.0-perf some parts of code are missing. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5ae98dbe
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5ae98dbe
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5ae98dbe
Branch: refs/heads/branch-3.0-perf
Commit: 5ae98dbeb1ad7752d02235a61d2869be2e284f55
Parents: 3051fa9
Author: Myroslav Papirkovskyi <mp...@hortonworks.com>
Authored: Tue Oct 10 18:37:05 2017 +0300
Committer: Myroslav Papirkovskyi <mp...@hortonworks.com>
Committed: Tue Oct 10 18:37:06 2017 +0300
----------------------------------------------------------------------
ambari-server/pom.xml | 6 +
.../actionmanager/ActionDBAccessorImpl.java | 11 +
.../server/actionmanager/ActionScheduler.java | 10 +-
.../actionmanager/ExecutionCommandWrapper.java | 121 +++---
.../ambari/server/agent/CommandRepository.java | 51 ++-
.../resources/ClusterResourceDefinition.java | 1 +
.../server/api/services/AmbariMetaInfo.java | 2 +-
.../ambari/server/checks/CheckDescription.java | 14 +-
.../checks/DatabaseConsistencyCheckHelper.java | 128 +++++-
.../checks/DruidHighAvailabilityCheck.java | 121 ++++++
.../checks/ServiceCheckValidityCheck.java | 84 ++--
.../AmbariCustomCommandExecutionHelper.java | 16 +-
.../AmbariManagementControllerImpl.java | 39 +-
.../server/controller/ConfigGroupRequest.java | 14 +-
.../server/controller/ControllerModule.java | 2 +
.../controller/DeleteIdentityHandler.java | 3 +-
.../server/controller/KerberosHelper.java | 8 +-
.../server/controller/KerberosHelperImpl.java | 13 +-
.../server/controller/RepositoryResponse.java | 23 +-
.../controller/ResourceProviderFactory.java | 4 +
.../AbstractControllerResourceProvider.java | 2 +-
.../internal/AbstractProviderModule.java | 15 +-
.../BlueprintConfigurationProcessor.java | 16 +-
.../internal/ClientConfigResourceProvider.java | 2 +
.../ClusterStackVersionResourceProvider.java | 21 +-
.../internal/ComponentResourceProvider.java | 4 +
.../internal/ConfigGroupResourceProvider.java | 8 +-
.../internal/DefaultProviderModule.java | 2 -
.../internal/RepositoryResourceProvider.java | 6 +
.../RepositoryVersionResourceProvider.java | 6 +-
.../internal/ServiceResourceProvider.java | 8 +
.../internal/UpgradeResourceProvider.java | 24 +-
.../VersionDefinitionResourceProvider.java | 4 +
.../server/events/AlertReceivedEvent.java | 8 +-
.../events/ClusterConfigFinishedEvent.java | 15 +-
.../DistributeRepositoriesActionListener.java | 8 +
.../listeners/upgrade/StackVersionListener.java | 12 +-
.../CachedRoleCommandOrderProvider.java | 2 +
.../ambari/server/orm/DBAccessorImpl.java | 6 +-
.../server/orm/dao/AlertDefinitionDAO.java | 13 +
.../ambari/server/orm/dao/ClusterDAO.java | 22 +-
.../server/orm/dao/HostRoleCommandDAO.java | 47 +++
.../ambari/server/orm/dao/ServiceConfigDAO.java | 2 +-
.../orm/entities/AlertDefinitionEntity.java | 4 +-
.../orm/entities/HostRoleCommandEntity.java | 64 ++-
.../server/orm/entities/RepositoryEntity.java | 22 +
.../orm/entities/RepositoryVersionEntity.java | 37 ++
.../server/orm/entities/UpgradeItemEntity.java | 2 +-
.../scheduler/ExecutionScheduleManager.java | 27 +-
.../AbstractPrepareKerberosServerAction.java | 14 +-
.../kerberos/CreateKeytabFilesServerAction.java | 9 +-
.../kerberos/KerberosIdentityDataFile.java | 2 +-
.../KerberosIdentityDataFileWriter.java | 9 +-
.../PrepareDisableKerberosServerAction.java | 2 +-
.../PrepareEnableKerberosServerAction.java | 2 +-
.../PrepareKerberosIdentitiesServerAction.java | 3 +-
.../upgrades/FinalizeUpgradeAction.java | 5 +
.../upgrades/PreconfigureKerberosAction.java | 6 +-
.../RangerUsersyncConfigCalculation.java | 96 +++++
.../apache/ambari/server/stack/RepoUtil.java | 4 +-
.../ambari/server/stack/StackContext.java | 118 ++++--
.../apache/ambari/server/stack/StackModule.java | 48 ++-
.../server/stack/StackServiceDirectory.java | 7 +-
.../ambari/server/state/RepositoryInfo.java | 27 +-
.../server/state/ServiceComponentImpl.java | 25 ++
.../ambari/server/state/ServiceOsSpecific.java | 16 +
.../apache/ambari/server/state/StackInfo.java | 1 -
.../ambari/server/state/UpgradeHelper.java | 109 ++++-
.../server/state/cluster/ClusterImpl.java | 4 +-
.../kerberos/AbstractKerberosDescriptor.java | 4 +-
.../AbstractKerberosDescriptorContainer.java | 25 +-
.../kerberos/KerberosComponentDescriptor.java | 2 +-
.../state/kerberos/KerberosDescriptor.java | 15 +-
.../kerberos/KerberosIdentityDescriptor.java | 78 +---
.../kerberos/KerberosKeytabDescriptor.java | 42 +-
.../kerberos/KerberosPrincipalDescriptor.java | 25 +-
.../kerberos/KerberosServiceDescriptor.java | 12 +-
.../services/AmbariServerAlertService.java | 21 +-
.../server/state/stack/RepoUrlInfoCallable.java | 217 ++++++++++
.../server/state/stack/RepoVdfCallable.java | 161 ++++++++
.../server/state/stack/RepositoryXml.java | 11 +
.../state/stack/upgrade/ConfigureTask.java | 3 +
.../stack/upgrade/RepositoryVersionHelper.java | 11 +
.../ambari/server/topology/AmbariContext.java | 6 +-
.../ambari/server/topology/TopologyManager.java | 96 +++--
.../topology/tasks/ConfigureClusterTask.java | 13 +-
.../tasks/ConfigureClusterTaskFactory.java | 3 +-
.../server/upgrade/UpgradeCatalog260.java | 345 +++++++++++++++-
.../server/utils/ManagedThreadPoolExecutor.java | 82 ++++
.../resources/Ambari-DDL-AzureDB-CREATE.sql | 3 +-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 3 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 3 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql | 3 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 3 +-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 3 +-
.../actionmanager/TestActionDBAccessorImpl.java | 63 ++-
.../server/agent/TestHeartbeatHandler.java | 5 +-
.../server/agent/TestHeartbeatMonitor.java | 20 +-
.../DatabaseConsistencyCheckHelperTest.java | 84 ++++
.../checks/DruidHighAvailabilityCheckTest.java | 176 ++++++++
.../checks/ServiceCheckValidityCheckTest.java | 92 +++--
.../AmbariManagementControllerTest.java | 14 +-
.../BlueprintConfigurationProcessorTest.java | 29 +-
...ClusterStackVersionResourceProviderTest.java | 232 ++++-------
.../internal/ComponentResourceProviderTest.java | 6 +
.../RepositoryResourceProviderTest.java | 20 +-
.../StackUpgradeConfigurationMergeTest.java | 136 +++++++
.../internal/UpgradeResourceProviderTest.java | 108 +++++
.../upgrade/StackVersionListenerTest.java | 46 +++
.../scheduler/ExecutionScheduleManagerTest.java | 43 +-
...AbstractPrepareKerberosServerActionTest.java | 2 +-
.../kerberos/KerberosIdentityDataFileTest.java | 10 +-
.../kerberos/KerberosServerActionTest.java | 2 +-
.../ComponentVersionCheckActionTest.java | 14 -
.../RangerUsersyncConfigCalculationTest.java | 126 ++++++
.../server/stack/KerberosDescriptorTest.java | 164 +++-----
.../server/stack/StackServiceDirectoryTest.java | 76 ++++
.../server/state/ServiceComponentTest.java | 21 +-
.../ambari/server/state/UpgradeHelperTest.java | 44 ++
.../KerberosComponentDescriptorTest.java | 20 +-
.../KerberosConfigurationDescriptorTest.java | 4 +-
.../state/kerberos/KerberosDescriptorTest.java | 54 +--
.../KerberosDescriptorUpdateHelperTest.java | 4 +-
.../KerberosIdentityDescriptorTest.java | 51 ++-
.../kerberos/KerberosKeytabDescriptorTest.java | 30 +-
.../KerberosPrincipalDescriptorTest.java | 26 +-
.../kerberos/KerberosServiceDescriptorTest.java | 26 +-
.../server/state/stack/UpgradePackTest.java | 1 +
.../ClusterDeployWithStartOnlyTest.java | 2 +-
...InstallWithoutStartOnComponentLevelTest.java | 2 +-
.../ClusterInstallWithoutStartTest.java | 2 +-
.../topology/ConfigureClusterTaskTest.java | 18 +-
.../server/topology/TopologyManagerTest.java | 2 +-
.../server/upgrade/UpgradeCatalog260Test.java | 398 +++++++++++++++----
.../utils/ManagedThreadPoolExecutorTest.java | 51 +++
135 files changed, 3948 insertions(+), 1053 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index de6ff51..bb0fa0a 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -1769,6 +1769,12 @@
<artifactId>jna</artifactId>
<version>4.2.2</version>
</dependency>
+ <dependency>
+ <groupId>com.networknt</groupId>
+ <artifactId>json-schema-validator</artifactId>
+ <version>0.1.10</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<pluginRepositories>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
index 555e2ad..f118c92 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java
@@ -552,6 +552,12 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
reportedTaskStatus = HostRoleStatus.SKIPPED_FAILED;
}
}
+
+ // if TIMEOUT and marked for holding then set status = HOLDING_TIMEOUT
+ if (reportedTaskStatus == HostRoleStatus.TIMEDOUT && commandEntity.isRetryAllowed()){
+ reportedTaskStatus = HostRoleStatus.HOLDING_TIMEDOUT;
+ }
+
if (!existingTaskStatus.isCompletedState()) {
commandEntity.setStatus(reportedTaskStatus);
}
@@ -615,6 +621,11 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
}
}
+ // if TIMEOUT and marked for holding then set status = HOLDING_TIMEOUT
+ if (status == HostRoleStatus.TIMEDOUT && command.isRetryAllowed()){
+ status = HostRoleStatus.HOLDING_TIMEDOUT;
+ }
+
command.setStatus(status);
command.setStdOut(report.getStdOut().getBytes());
command.setStdError(report.getStdErr().getBytes());
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
index c41dd01..5c830ba 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
@@ -927,12 +927,14 @@ class ActionScheduler implements Runnable {
RoleCommandPair roleCommand = new
RoleCommandPair(Role.valueOf(command.getRole()), command.getRoleCommand());
Set<RoleCommandPair> roleCommandDependencies = rco.getDependencies().get(roleCommand);
- // remove eventual references to the same RoleCommand
- roleCommandDependencies.remove(roleCommand);
// check if there are any dependencies IN_PROGRESS
- if (roleCommandDependencies != null && CollectionUtils.containsAny(rolesCommandsInProgress, roleCommandDependencies)) {
- areCommandDependenciesFinished = false;
+ if (roleCommandDependencies != null) {
+ // remove eventual references to the same RoleCommand
+ roleCommandDependencies.remove(roleCommand);
+ if (CollectionUtils.containsAny(rolesCommandsInProgress, roleCommandDependencies)) {
+ areCommandDependenciesFinished = false;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 903619e..79ca5ba 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -19,6 +19,7 @@ package org.apache.ambari.server.actionmanager;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
import java.util.Map;
import java.util.TreeMap;
@@ -29,7 +30,6 @@ import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.ExecutionCommand;
-import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
@@ -167,88 +167,93 @@ public class ExecutionCommandWrapper {
configHelper.getAndMergeHostConfigAttributes(executionCommand.getConfigurationAttributes(),
configurationTags, cluster);
- // set the repository version for the component this command is for -
- // always use the current desired version
- try {
- RepositoryVersionEntity repositoryVersion = null;
- String serviceName = executionCommand.getServiceName();
- if (!StringUtils.isEmpty(serviceName)) {
- Service service = cluster.getService(serviceName);
- if (null != service) {
- repositoryVersion = service.getDesiredRepositoryVersion();
- }
+ setVersions(cluster);
+
+ // provide some basic information about a cluster upgrade if there is one
+ // in progress
+ UpgradeEntity upgrade = cluster.getUpgradeInProgress();
+ if (null != upgrade) {
+ UpgradeContext upgradeContext = upgradeContextFactory.create(cluster, upgrade);
+ UpgradeSummary upgradeSummary = upgradeContext.getUpgradeSummary();
+ executionCommand.setUpgradeSummary(upgradeSummary);
+ }
+
+ } catch (ClusterNotFoundException cnfe) {
+ // it's possible that there are commands without clusters; in such cases,
+ // just return the de-serialized command and don't try to read configs
+ LOG.warn(
+ "Unable to lookup the cluster by ID; assuming that there is no cluster and therefore no configs for this execution command: {}",
+ cnfe.getMessage());
+
+ return executionCommand;
+ } catch (AmbariException e) {
+ throw new RuntimeException(e);
+ }
+
+ return executionCommand;
+ }
+
+ public void setVersions(Cluster cluster) {
+ // set the repository version for the component this command is for -
+ // always use the current desired version
+ String serviceName = executionCommand.getServiceName();
+ try {
+ RepositoryVersionEntity repositoryVersion = null;
+ if (!StringUtils.isEmpty(serviceName)) {
+ Service service = cluster.getService(serviceName);
+ if (null != service) {
+ repositoryVersion = service.getDesiredRepositoryVersion();
String componentName = executionCommand.getComponentName();
if (!StringUtils.isEmpty(componentName)) {
- ServiceComponent serviceComponent = service.getServiceComponent(
- executionCommand.getComponentName());
-
+ ServiceComponent serviceComponent = service.getServiceComponent(componentName);
if (null != serviceComponent) {
repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
}
}
}
+ }
- Map<String, String> commandParams = executionCommand.getCommandParams();
+ Map<String, String> commandParams = executionCommand.getCommandParams();
- if (null != repositoryVersion) {
- // only set the version if it's not set and this is NOT an install
- // command
- if (!commandParams.containsKey(KeyNames.VERSION)
- && executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
- commandParams.put(KeyNames.VERSION, repositoryVersion.getVersion());
- }
+ if (null != repositoryVersion) {
+ // only set the version if it's not set and this is NOT an install
+ // command
+ if (!commandParams.containsKey(VERSION)
+ && executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
+ commandParams.put(VERSION, repositoryVersion.getVersion());
+ }
- StackId stackId = repositoryVersion.getStackId();
- StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
- stackId.getStackVersion());
+ StackId stackId = repositoryVersion.getStackId();
+ StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
+ stackId.getStackVersion());
- if (!commandParams.containsKey(HOOKS_FOLDER)) {
- commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
- }
+ if (!commandParams.containsKey(HOOKS_FOLDER)) {
+ commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+ }
- if (!commandParams.containsKey(SERVICE_PACKAGE_FOLDER)) {
- if (!StringUtils.isEmpty(serviceName)) {
- ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
- stackId.getStackVersion(), serviceName);
+ if (!commandParams.containsKey(SERVICE_PACKAGE_FOLDER)) {
+ if (!StringUtils.isEmpty(serviceName)) {
+ ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
+ stackId.getStackVersion(), serviceName);
- commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
- }
+ commandParams.put(SERVICE_PACKAGE_FOLDER, serviceInfo.getServicePackageFolder());
}
}
- } catch (ServiceNotFoundException serviceNotFoundException) {
- // it's possible that there are commands specified for a service where
- // the service doesn't exist yet
- LOG.warn(
- "The service {} is not installed in the cluster. No repository version will be sent for this command.",
- executionCommand.getServiceName());
}
// set the desired versions of versionable components. This is safe even during an upgrade because
// we are "loading-late": components that have not yet upgraded in an EU will have the correct versions.
executionCommand.setComponentVersions(cluster);
-
- // provide some basic information about a cluster upgrade if there is one
- // in progress
- UpgradeEntity upgrade = cluster.getUpgradeInProgress();
- if (null != upgrade) {
- UpgradeContext upgradeContext = upgradeContextFactory.create(cluster, upgrade);
- UpgradeSummary upgradeSummary = upgradeContext.getUpgradeSummary();
- executionCommand.setUpgradeSummary(upgradeSummary);
- }
- } catch (ClusterNotFoundException cnfe) {
- // it's possible that there are commands without clusters; in such cases,
- // just return the de-serialized command and don't try to read configs
+ } catch (ServiceNotFoundException serviceNotFoundException) {
+ // it's possible that there are commands specified for a service where
+ // the service doesn't exist yet
LOG.warn(
- "Unable to lookup the cluster by ID; assuming that there is no cluster and therefore no configs for this execution command: {}",
- cnfe.getMessage());
-
- return executionCommand;
+ "The service {} is not installed in the cluster. No repository version will be sent for this command.",
+ serviceName);
} catch (AmbariException e) {
throw new RuntimeException(e);
}
-
- return executionCommand;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
index 858a55f..301f475 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
@@ -46,6 +46,13 @@ public class CommandRepository {
private String m_stackName;
/**
+ * {@code true} if Ambari believes that this repository has reported back it's
+ * version after distribution.
+ */
+ @SerializedName("resolved")
+ private boolean m_resolved;
+
+ /**
* @param version the repo version
*/
public void setRepositoryVersion(String version) {
@@ -119,6 +126,27 @@ public class CommandRepository {
}
/**
+ * Gets whether this repository has been marked as having its version
+ * resolved.
+ *
+ * @return {@code true} if this repository has been confirmed to have the
+ * right version.
+ */
+ public boolean isResolved() {
+ return m_resolved;
+ }
+
+ /**
+ * Gets whether this repository has had its version resolved.
+ *
+ * @param resolved
+ * {@code true} to mark this repository as being resolved.
+ */
+ public void setResolved(boolean resolved) {
+ m_resolved = resolved;
+ }
+
+ /**
* Minimal information required to generate repo files on the agent. These are copies
* of the repository objects from repo versions that can be changed for URL overrides, etc.
*/
@@ -133,12 +161,16 @@ public class CommandRepository {
@SerializedName("ambariManaged")
private boolean m_ambariManaged = true;
- /**
- * The name should not change. Ubuntu requires that it match exactly as the repo was built.
- */
+
@SerializedName("repoName")
private final String m_repoName;
+ @SerializedName("distribution")
+ private final String m_distribution;
+
+ @SerializedName("components")
+ private final String m_components;
+
@SerializedName("mirrorsList")
private String m_mirrorsList;
@@ -149,6 +181,8 @@ public class CommandRepository {
m_osType = info.getOsType();
m_repoId = info.getRepoId();
m_repoName = info.getRepoName();
+ m_distribution = info.getDistribution();
+ m_components = info.getComponents();
m_mirrorsList = info.getMirrorsList();
}
@@ -156,6 +190,8 @@ public class CommandRepository {
m_baseUrl = entity.getBaseUrl();
m_repoId = entity.getRepositoryId();
m_repoName = entity.getName();
+ m_distribution = entity.getDistribution();
+ m_components = entity.getComponents();
m_mirrorsList = entity.getMirrorsList();
m_osType = osType;
}
@@ -176,6 +212,13 @@ public class CommandRepository {
return m_repoName;
}
+ public String getDistribution() {
+ return m_distribution;
+ }
+
+ public String getComponents() {
+ return m_components;
+ }
public String getBaseUrl() {
return m_baseUrl;
@@ -193,6 +236,8 @@ public class CommandRepository {
return new ToStringBuilder(null)
.append("os", m_osType)
.append("name", m_repoName)
+ .append("distribution", m_distribution)
+ .append("components", m_components)
.append("id", m_repoId)
.append("baseUrl", m_baseUrl)
.toString();
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
index 8933dd3..9d0c169 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java
@@ -87,6 +87,7 @@ public class ClusterResourceDefinition extends BaseResourceDefinition {
directives.add(KerberosHelper.DIRECTIVE_FORCE_TOGGLE_KERBEROS);
directives.add(KerberosHelper.DIRECTIVE_HOSTS);
directives.add(KerberosHelper.DIRECTIVE_COMPONENTS);
+ directives.add(KerberosHelper.DIRECTIVE_IGNORE_CONFIGS);
return directives;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 1787b49..2a794de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@ -1363,7 +1363,7 @@ public class AmbariMetaInfo {
/**
* Ensures that the map of version definition files is populated
*/
- private void ensureVersionDefinitions() {
+ private synchronized void ensureVersionDefinitions() {
if (null != versionDefinitions) {
return;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
index 4fd37dc..314ab83 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java
@@ -341,6 +341,18 @@ public class CheckDescription {
.put(AbstractCheckDescriptor.DEFAULT, "The following components do not exist in the target repository's stack. They must be removed from the cluster before upgrading.")
.build());
+ public static CheckDescription DRUID_HA_WARNING = new CheckDescription(
+ "DRUID_HA",
+ PrereqCheckType.SERVICE,
+ "Druid Downtime During Upgrade",
+ new ImmutableMap.Builder<String, String>()
+ .put(
+ AbstractCheckDescriptor.DEFAULT,
+ "High Availability is not enabled for Druid. Druid Service may have some downtime during upgrade. Deploy multiple instances of %s in the Cluster to avoid any downtime."
+ )
+ .build()
+ );
+
private String m_name;
private PrereqCheckType m_type;
private String m_description;
@@ -380,4 +392,4 @@ public class CheckDescription {
public String getFail(String key) {
return m_fails.containsKey(key) ? m_fails.get(key) : "";
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index 054c470..34888f2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -40,6 +40,7 @@ import java.util.regex.Pattern;
import javax.annotation.Nullable;
import javax.inject.Provider;
import javax.persistence.EntityManager;
+import javax.persistence.Query;
import javax.persistence.TypedQuery;
import org.apache.ambari.server.AmbariException;
@@ -59,6 +60,8 @@ import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
import org.apache.ambari.server.orm.entities.MetainfoEntity;
import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
import org.apache.ambari.server.state.ClientConfigFileDefinition;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.State;
@@ -92,6 +95,10 @@ public class DatabaseConsistencyCheckHelper {
private static StageDAO stageDAO;
private static DatabaseConsistencyCheckResult checkResult = DatabaseConsistencyCheckResult.DB_CHECK_SUCCESS;
+ public static final String GET_CONFIGS_SELECTED_MORE_THAN_ONCE_QUERY = "select c.cluster_name, cc.type_name from clusterconfig cc " +
+ "join clusters c on cc.cluster_id=c.cluster_id " +
+ "group by c.cluster_name, cc.type_name " +
+ "having sum(cc.selected) > 1";
/**
* @return The result of the DB cheks run so far.
@@ -174,6 +181,7 @@ public class DatabaseConsistencyCheckHelper {
if (fixIssues) {
fixHostComponentStatesCountEqualsHostComponentsDesiredStates();
fixClusterConfigsNotMappedToAnyService();
+ fixConfigsSelectedMoreThanOnce();
}
checkSchemaName();
checkMySQLEngine();
@@ -317,7 +325,7 @@ public class DatabaseConsistencyCheckHelper {
warning("Unable to get size for table {}!", tableName);
}
} catch (SQLException ex) {
- error(String.format("Failed to get %s row count: ", tableName), e);
+ warning(String.format("Failed to get %s row count: ", tableName), e);
}
} finally {
if (rs != null) {
@@ -376,7 +384,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- error("Exception occurred during check for config selected more than once procedure: ", e);
+ warning("Exception occurred during check for config selected more than once procedure: ", e);
} finally {
if (rs != null) {
try {
@@ -420,12 +428,12 @@ public class DatabaseConsistencyCheckHelper {
}
if (!hostsWithoutStatus.isEmpty()) {
- error("You have host(s) without state (in hoststate table): " + StringUtils.join(hostsWithoutStatus, ","));
+ warning("You have host(s) without state (in hoststate table): " + StringUtils.join(hostsWithoutStatus, ","));
}
}
} catch (SQLException e) {
- error("Exception occurred during check for host without state procedure: ", e);
+ warning("Exception occurred during check for host without state procedure: ", e);
} finally {
if (rs != null) {
try {
@@ -458,7 +466,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- error("Exception occurred during topology request tables check: ", e);
+ warning("Exception occurred during topology request tables check: ", e);
} finally {
if (rs != null) {
try {
@@ -523,7 +531,7 @@ public class DatabaseConsistencyCheckHelper {
}
if (hostComponentStateCount != hostComponentDesiredStateCount || hostComponentStateCount != mergedCount) {
- error("Your host component states (hostcomponentstate table) count not equals host component desired states (hostcomponentdesiredstate table) count!");
+ warning("Your host component states (hostcomponentstate table) count not equals host component desired states (hostcomponentdesiredstate table) count!");
}
@@ -535,11 +543,11 @@ public class DatabaseConsistencyCheckHelper {
}
for (Map.Entry<String, String> component : hostComponentStateDuplicates.entrySet()) {
- error("Component {} on host with id {}, has more than one host component state (hostcomponentstate table)!", component.getKey(), component.getValue());
+ warning("Component {} on host with id {}, has more than one host component state (hostcomponentstate table)!", component.getKey(), component.getValue());
}
} catch (SQLException e) {
- error("Exception occurred during check for same count of host component states and host component desired states: ", e);
+ warning("Exception occurred during check for same count of host component states and host component desired states: ", e);
} finally {
if (rs != null) {
try {
@@ -774,11 +782,11 @@ public class DatabaseConsistencyCheckHelper {
tablesInfo.add(rs.getString("TABLE_NAME"));
}
if (!tablesInfo.isEmpty()){
- error("Found tables with engine type that is not InnoDB : {}", tablesInfo);
+ warning("Found tables with engine type that is not InnoDB : {}", tablesInfo);
}
}
} catch (SQLException e) {
- error("Exception occurred during checking MySQL engine to be innodb: ", e);
+ warning("Exception occurred during checking MySQL engine to be innodb: ", e);
} finally {
if (rs != null) {
try {
@@ -791,12 +799,102 @@ public class DatabaseConsistencyCheckHelper {
}
/**
+ * Fix inconsistencies found by {@code checkForConfigsSelectedMoreThanOnce}
+ * selecting latest one by selectedTimestamp
+ */
+ @Transactional
+ static void fixConfigsSelectedMoreThanOnce() {
+ LOG.info("Fix configs selected more than once");
+ ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class);
+
+ Clusters clusters = injector.getInstance(Clusters.class);
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+
+ Multimap<String, String> clusterConfigTypeMap = HashMultimap.create();
+ ResultSet rs = null;
+ Statement statement = null;
+
+ ensureConnection();
+
+ try {
+ statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE);
+ rs = statement.executeQuery(GET_CONFIGS_SELECTED_MORE_THAN_ONCE_QUERY);
+ if (rs != null) {
+ while (rs.next()) {
+ clusterConfigTypeMap.put(rs.getString("cluster_name"), rs.getString("type_name"));
+ }
+ }
+
+ } catch (SQLException e) {
+ warning("Exception occurred during check for config selected more than once procedure: ", e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (SQLException e) {
+ LOG.error("Exception occurred during result set closing procedure: ", e);
+ }
+ }
+
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ LOG.error("Exception occurred during statement closing procedure: ", e);
+ }
+ }
+ }
+
+ for (String clusterName : clusterConfigTypeMap.keySet()) {
+ Cluster cluster = null;
+ try {
+ cluster = clusters.getCluster(clusterName);
+
+ Collection<String> typesWithMultipleSelectedConfigs = clusterConfigTypeMap.get(clusterName);
+
+ for (String type: typesWithMultipleSelectedConfigs) {
+ List<ClusterConfigEntity> enabledConfigsByType = getEnabledConfigsByType(cluster.getClusterId(), type);
+ ClusterConfigEntity latestConfig = enabledConfigsByType.get(0);
+ for (ClusterConfigEntity entity : enabledConfigsByType){
+ entity.setSelected(false);
+ if (latestConfig.getSelectedTimestamp() < entity.getSelectedTimestamp()){
+ latestConfig = entity;
+ }
+ clusterDAO.merge(entity, true);
+ }
+ latestConfig.setSelected(true);
+ clusterDAO.merge(latestConfig, true);
+ }
+ } catch (AmbariException e) {
+ warning("Exception occurred during fix for config selected more than once procedure: ", e);
+ }
+ }
+ }
+
+ /**
+ * Find ClusterConfigs with selected = 1
+ * @return ClusterConfigs that are not mapped to Service by type
+ */
+ private static List<ClusterConfigEntity> getEnabledConfigsByType(long clusterId, String type) {
+
+ Provider<EntityManager> entityManagerProvider = injector.getProvider(EntityManager.class);
+ EntityManager entityManager = entityManagerProvider.get();
+
+ Query query = entityManager.createNamedQuery("ClusterConfigEntity.findEnabledConfigByType",ClusterConfigEntity.class);
+ query.setParameter("clusterId", clusterId);
+ query.setParameter("type", type);
+
+ return (List<ClusterConfigEntity>) query.getResultList();
+ }
+
+ /**
* This method checks several potential problems for services:
* 1) Check if we have services in cluster which doesn't have service config id(not available in serviceconfig table).
* 2) Check if service has no mapped configs to it's service config id.
* 3) Check if service has all required configs mapped to it.
* 4) Check if service has config which is not selected(has no actual config version)
- * If any issue was discovered, we are showing error message for user.
+ * If any issue was discovered, we are showing warning message for user.
* */
static void checkServiceConfigs() {
LOG.info("Checking services and their configs");
@@ -875,7 +973,7 @@ public class DatabaseConsistencyCheckHelper {
for (String clName : clusterServiceVersionMap.keySet()) {
Multimap<String, String> serviceVersion = clusterServiceVersionMap.get(clName);
for (String servName : serviceVersion.keySet()) {
- error("In cluster {}, service config mapping is unavailable (in table serviceconfigmapping) for service {} with version(s) {}! ", clName, servName, StringUtils.join(serviceVersion.get(servName), ","));
+ warning("In cluster {}, service config mapping is unavailable (in table serviceconfigmapping) for service {} with version(s) {}! ", clName, servName, StringUtils.join(serviceVersion.get(servName), ","));
}
}
@@ -986,7 +1084,7 @@ public class DatabaseConsistencyCheckHelper {
}
if (!serviceConfigsFromStack.isEmpty()) {
- error("Required config(s): {} is(are) not available for service {} with service config version {} in cluster {}",
+ warning("Required config(s): {} is(are) not available for service {} with service config version {} in cluster {}",
StringUtils.join(serviceConfigsFromStack, ","), serviceName, Integer.toString(serviceVersion), clusterName);
}
}
@@ -1024,11 +1122,11 @@ public class DatabaseConsistencyCheckHelper {
for (String clusterName : clusterServiceConfigType.keySet()) {
Multimap<String, String> serviceConfig = clusterServiceConfigType.get(clusterName);
for (String serviceName : serviceConfig.keySet()) {
- error("You have non selected configs: {} for service {} from cluster {}!", StringUtils.join(serviceConfig.get(serviceName), ","), serviceName, clusterName);
+ warning("You have non selected configs: {} for service {} from cluster {}!", StringUtils.join(serviceConfig.get(serviceName), ","), serviceName, clusterName);
}
}
} catch (SQLException | AmbariException e) {
- error("Exception occurred during complex service check procedure: ", e);
+ warning("Exception occurred during complex service check procedure: ", e);
} finally {
if (rs != null) {
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/checks/DruidHighAvailabilityCheck.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DruidHighAvailabilityCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DruidHighAvailabilityCheck.java
new file mode 100644
index 0000000..88502ba
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DruidHighAvailabilityCheck.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.checks;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.ServiceComponentNotFoundException;
+import org.apache.ambari.server.controller.PrereqCheckRequest;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.stack.PrereqCheckStatus;
+import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.inject.Singleton;
+
+/**
+ * Checks that namenode high availability is enabled.
+ */
+@Singleton
+@UpgradeCheck(group = UpgradeCheckGroup.MULTIPLE_COMPONENT_WARNING, order = 16.0f)
+public class DruidHighAvailabilityCheck extends AbstractCheckDescriptor
+{
+
+ public static final String DRUID_SERVICE_NAME = "DRUID";
+ public static final String[] DRUID_COMPONENT_NAMES = new String[]{
+ "DRUID_BROKER",
+ "DRUID_COORDINATOR",
+ "DRUID_HISTORICAL",
+ "DRUID_OVERLORD",
+ "DRUID_MIDDLEMANAGER",
+ "DRUID_ROUTER"
+ };
+
+ /**
+ * Constructor.
+ */
+ public DruidHighAvailabilityCheck()
+ {
+ super(CheckDescription.DRUID_HA_WARNING);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Set<String> getApplicableServices()
+ {
+ return Sets.newHashSet(DRUID_SERVICE_NAME);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<CheckQualification> getQualifications()
+ {
+ return Arrays.asList(
+ new PriorCheckQualification(CheckDescription.DRUID_HA_WARNING));
+ }
+
+ @Override
+ public void perform(PrerequisiteCheck prerequisiteCheck, PrereqCheckRequest request) throws AmbariException
+ {
+ List<String> haNotEnabledComponents = Lists.newArrayList();
+ for (String component : DRUID_COMPONENT_NAMES) {
+ Set<String> hosts = getHostsForComponent(request, component);
+ if (hosts.size() == 1) {
+ // This component is installed on only 1 host, HA is not enabled for it.
+ haNotEnabledComponents.add(component);
+ }
+ }
+ if (!haNotEnabledComponents.isEmpty()) {
+ prerequisiteCheck.getFailedOn().add(DRUID_SERVICE_NAME);
+ prerequisiteCheck.setStatus(PrereqCheckStatus.WARNING);
+ String failReason = getFailReason(prerequisiteCheck, request);
+ prerequisiteCheck.setFailReason(String.format(failReason, StringUtils.join(haNotEnabledComponents.toArray(), ", ")));
+ }
+
+ }
+
+ private Set<String> getHostsForComponent(PrereqCheckRequest request, String componentName)
+ throws AmbariException
+ {
+ Set<String> hosts = new HashSet<>();
+ final String clusterName = request.getClusterName();
+ final Cluster cluster = clustersProvider.get().getCluster(clusterName);
+ try {
+ ServiceComponent serviceComponent = cluster.getService(DRUID_SERVICE_NAME).getServiceComponent(componentName);
+ if (serviceComponent != null) {
+ hosts = serviceComponent.getServiceComponentHosts().keySet();
+ }
+ }
+ catch (ServiceComponentNotFoundException err) {
+ // This exception can be ignored if the component doesn't exist because it is a best-attempt at finding it.
+ }
+
+ return hosts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java
index ad68a2c..a4c2430 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java
@@ -19,29 +19,19 @@ package org.apache.ambari.server.checks;
import java.text.SimpleDateFormat;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.Role;
-import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.controller.PrereqCheckRequest;
-import org.apache.ambari.server.controller.internal.PageRequestImpl;
-import org.apache.ambari.server.controller.internal.RequestImpl;
-import org.apache.ambari.server.controller.internal.SortRequestImpl;
-import org.apache.ambari.server.controller.internal.TaskResourceProvider;
-import org.apache.ambari.server.controller.spi.PageRequest;
-import org.apache.ambari.server.controller.spi.Predicate;
-import org.apache.ambari.server.controller.spi.SortRequest;
-import org.apache.ambari.server.controller.spi.SortRequestProperty;
-import org.apache.ambari.server.controller.utilities.PredicateBuilder;
+import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.HostRoleCommandDAO.LastServiceCheckDTO;
import org.apache.ambari.server.orm.dao.ServiceConfigDAO;
-import org.apache.ambari.server.orm.entities.HostRoleCommandEntity;
import org.apache.ambari.server.orm.entities.ServiceConfigEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.MaintenanceState;
@@ -73,17 +63,6 @@ public class ServiceCheckValidityCheck extends AbstractCheckDescriptor {
private static final Logger LOG = LoggerFactory.getLogger(ServiceCheckValidityCheck.class);
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("MM-dd-yyyy hh:mm:ss");
- private static List<SortRequestProperty> sortRequestProperties =
- Collections.singletonList(new SortRequestProperty(TaskResourceProvider.TASK_START_TIME_PROPERTY_ID, SortRequest.Order.DESC));
- private static SortRequest sortRequest = new SortRequestImpl(sortRequestProperties);
- private static final PageRequestImpl PAGE_REQUEST = new PageRequestImpl(PageRequest.StartingPoint.End, 1000, 0, null, null);
- private static final RequestImpl REQUEST = new RequestImpl(null, null, null, null, sortRequest, PAGE_REQUEST);
- private static final Predicate PREDICATE = new PredicateBuilder()
- .property(TaskResourceProvider.TASK_COMMAND_PROPERTY_ID).equals(RoleCommand.SERVICE_CHECK.name())
- .and().property(TaskResourceProvider.TASK_START_TIME_PROPERTY_ID).greaterThan(-1)
- .toPredicate();
-
-
@Inject
Provider<ServiceConfigDAO> serviceConfigDAOProvider;
@@ -91,6 +70,8 @@ public class ServiceCheckValidityCheck extends AbstractCheckDescriptor {
@Inject
Provider<HostRoleCommandDAO> hostRoleCommandDAOProvider;
+ @Inject
+ Provider<ActionMetadata> actionMetadataProvider;
/**
* Constructor.
@@ -113,8 +94,8 @@ public class ServiceCheckValidityCheck extends AbstractCheckDescriptor {
final Cluster cluster = clustersProvider.get().getCluster(clusterName);
long clusterId = cluster.getClusterId();
+ // build a mapping of the last config changes by service
Map<String, Long> lastServiceConfigUpdates = new HashMap<>();
-
for (Service service : cluster.getServices().values()) {
if (service.getMaintenanceState() != MaintenanceState.OFF || !hasAtLeastOneComponentVersionAdvertised(service)) {
continue;
@@ -130,43 +111,34 @@ public class ServiceCheckValidityCheck extends AbstractCheckDescriptor {
}
}
- List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findAll(REQUEST, PREDICATE);
-
- // !!! build a map of Role to latest-config-check in case it was rerun multiple times, we want the latest
- Map<Role, HostRoleCommandEntity> latestTimestamps = new HashMap<>();
- for (HostRoleCommandEntity command : commands) {
- Role role = command.getRole();
-
- // Because results are already sorted by start_time desc, first occurrence is guaranteed to have max(start_time).
- if (!latestTimestamps.containsKey(role)) {
- latestTimestamps.put(role, command);
- }
+ // get the latest service checks, grouped by role
+ List<LastServiceCheckDTO> lastServiceChecks = hostRoleCommandDAO.getLatestServiceChecksByRole(clusterId);
+ Map<String, Long> lastServiceChecksByRole = new HashMap<>();
+ for( LastServiceCheckDTO lastServiceCheck : lastServiceChecks ) {
+ lastServiceChecksByRole.put(lastServiceCheck.role, lastServiceCheck.endTime);
}
LinkedHashSet<String> failedServiceNames = new LinkedHashSet<>();
- for (Map.Entry<String, Long> serviceEntry : lastServiceConfigUpdates.entrySet()) {
- String serviceName = serviceEntry.getKey();
- Long configTimestamp = serviceEntry.getValue();
-
- boolean serviceCheckWasExecuted = false;
- for (HostRoleCommandEntity command : latestTimestamps.values()) {
- if (null != command.getCommandDetail() && command.getCommandDetail().contains(serviceName)) {
- serviceCheckWasExecuted = true;
- Long serviceCheckTimestamp = command.getStartTime();
-
- if (serviceCheckTimestamp < configTimestamp) {
- failedServiceNames.add(serviceName);
- LOG.info("Service {} latest config change is {}, latest service check executed at {}",
- serviceName,
- DATE_FORMAT.format(new Date(configTimestamp)),
- DATE_FORMAT.format(new Date(serviceCheckTimestamp)));
- }
- }
+
+ // for every service, see if there was a service check executed and then
+ for( Entry<String, Long> entry : lastServiceConfigUpdates.entrySet() ) {
+ String serviceName = entry.getKey();
+ long configCreationTime = entry.getValue();
+ String role = actionMetadataProvider.get().getServiceCheckAction(serviceName);
+
+ if(!lastServiceChecksByRole.containsKey(role) ) {
+ LOG.info("There was no service check found for service {} matching role {}", serviceName, role);
+ failedServiceNames.add(serviceName);
+ continue;
}
- if (!serviceCheckWasExecuted) {
+ long lastServiceCheckTime = lastServiceChecksByRole.get(role);
+ if (lastServiceCheckTime < configCreationTime) {
failedServiceNames.add(serviceName);
- LOG.info("Service {} service check has never been executed", serviceName);
+ LOG.info(
+ "The {} service (role {}) had its configurations updated on {}, but the last service check was {}",
+ serviceName, role, DATE_FORMAT.format(new Date(configCreationTime)),
+ DATE_FORMAT.format(new Date(lastServiceCheckTime)));
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index d3e1488..5ffdf26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -1312,14 +1312,18 @@ public class AmbariCustomCommandExecutionHelper {
if (actionName.equals(START_COMMAND_NAME) || actionName.equals(RESTART_COMMAND_NAME)) {
Cluster cluster = clusters.getCluster(clusterName);
StackId stackId = null;
- try {
- Service service = cluster.getService(serviceName);
- stackId = service.getDesiredStackId();
- } catch (AmbariException e) {
- LOG.debug("Could not load service {}, skipping topology check", serviceName);
- stackId = cluster.getDesiredStackVersion();
+ if (serviceName != null) {
+ try {
+ Service service = cluster.getService(serviceName);
+ stackId = service.getDesiredStackId();
+ } catch (AmbariException e) {
+ LOG.debug("Could not load service {}, skipping topology check", serviceName);
+ }
}
+ if (stackId == null) {
+ stackId = cluster.getDesiredStackVersion();
+ }
AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 817f340..9db5832 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -97,6 +97,7 @@ import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.StackAccessException;
import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.actionmanager.CommandExecutionType;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
@@ -2451,8 +2452,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
stackId.getStackVersion());
Map<String, ServiceInfo> servicesMap = ambariMetaInfo.getServices(stackInfo.getName(), stackInfo.getVersion());
- ExecutionCommand execCmd = stage.getExecutionCommandWrapper(scHost.getHostName(),
- scHost.getServiceComponentName()).getExecutionCommand();
+ ExecutionCommandWrapper execCmdWrapper = stage.getExecutionCommandWrapper(hostname, componentName);
+ ExecutionCommand execCmd = execCmdWrapper.getExecutionCommand();
execCmd.setConfigurations(configurations);
execCmd.setConfigurationAttributes(configurationAttributes);
@@ -3055,15 +3056,22 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
break;
case INIT:
- throw new AmbariException("Unsupported transition to INIT for"
- + " servicecomponenthost"
- + ", clusterName=" + cluster.getClusterName()
- + ", clusterId=" + cluster.getClusterId()
- + ", serviceName=" + scHost.getServiceName()
- + ", componentName=" + scHost.getServiceComponentName()
- + ", hostname=" + scHost.getHostName()
- + ", currentState=" + oldSchState
- + ", newDesiredState=" + newState);
+ if (oldSchState == State.INSTALLED ||
+ oldSchState == State.INSTALL_FAILED ||
+ oldSchState == State.INIT) {
+ scHost.setState(State.INIT);
+ continue;
+ } else {
+ throw new AmbariException("Unsupported transition to INIT for"
+ + " servicecomponenthost"
+ + ", clusterName=" + cluster.getClusterName()
+ + ", clusterId=" + cluster.getClusterId()
+ + ", serviceName=" + scHost.getServiceName()
+ + ", componentName=" + scHost.getServiceComponentName()
+ + ", hostname=" + scHost.getHostName()
+ + ", currentState=" + oldSchState
+ + ", newDesiredState=" + newState);
+ }
default:
throw new AmbariException("Unsupported state change operation"
+ ", newState=" + newState);
@@ -3716,6 +3724,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
for (ServiceComponentHost componentHost : entry.getValue()) {
try {
//actually delete the component
+ //TODO update metadata processing according to AMBARI-21587
entry.getKey().deleteServiceComponentHosts(componentHost.getHostName(), deleteMetaData);
//create cluster-master-service map to update all include/exclude files in one action
@@ -3817,7 +3826,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
LOG.debug("Refresh include/exclude files action will be executed for " + serviceMasterMap);
HashMap<String, String> requestProperties = new HashMap<>();
requestProperties.put("context", "Update Include/Exclude Files for " + serviceMasterMap.keySet().toString());
- requestProperties.put("exclusive", "true");
HashMap<String, String> params = new HashMap<>();
params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY, String.valueOf(isDecommission));
@@ -3838,7 +3846,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
//Create request for command
ExecuteActionRequest actionRequest = new ExecuteActionRequest(
clusterName, AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null,
- resourceFilters, null, params, true);
+ resourceFilters, null, params, false);
//Send action
createAction(actionRequest, requestProperties);
}
@@ -4572,7 +4580,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
for (OperatingSystemEntity operatingSystem: repositoryVersion.getOperatingSystems()) {
if (operatingSystem.getOsType().equals(osType)) {
for (RepositoryEntity repository: operatingSystem.getRepositories()) {
- final RepositoryResponse response = new RepositoryResponse(repository.getBaseUrl(), osType, repository.getRepositoryId(), repository.getName(), "", "");
+ final RepositoryResponse response = new RepositoryResponse(repository.getBaseUrl(), osType, repository.getRepositoryId(),
+ repository.getName(), repository.getDistribution(), repository.getComponents(), "", "");
if (null != versionDefinitionId) {
response.setVersionDefinitionId(versionDefinitionId);
} else {
@@ -4600,7 +4609,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
for (RepositoryXml.Repo repo : os.getRepos()) {
RepositoryResponse resp = new RepositoryResponse(repo.getBaseUrl(), os.getFamily(),
- repo.getRepoId(), repo.getRepoName(), repo.getMirrorsList(),
+ repo.getRepoId(), repo.getRepoName(), repo.getDistribution(), repo.getComponents(), repo.getMirrorsList(),
repo.getBaseUrl());
resp.setVersionDefinitionId(versionDefinitionId);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigGroupRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigGroupRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigGroupRequest.java
index cb20328..babdf10 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigGroupRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigGroupRequest.java
@@ -27,18 +27,20 @@ public class ConfigGroupRequest {
private String clusterName;
private String groupName;
private String tag;
+ private String serviceName;
private String description;
private String serviceConfigVersionNote;
private Set<String> hosts;
private Map<String, Config> configs;
public ConfigGroupRequest(Long id, String clusterName, String groupName,
- String tag, String description, Set<String> hosts,
- Map<String, Config> configs) {
+ String tag, String serviceName, String description,
+ Set<String> hosts, Map<String, Config> configs) {
this.id = id;
this.clusterName = clusterName;
this.groupName = groupName;
this.tag = tag;
+ this.serviceName = serviceName;
this.description = description;
this.hosts = hosts;
this.configs = configs;
@@ -68,6 +70,14 @@ public class ConfigGroupRequest {
this.tag = tag;
}
+ public String getServiceName() {
+ return serviceName;
+ }
+
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
public String getDescription() {
return description;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
index 5981287..6d72855 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
@@ -62,6 +62,7 @@ import org.apache.ambari.server.cleanup.ClasspathScannerUtils;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.configuration.Configuration.ConnectionPoolType;
import org.apache.ambari.server.configuration.Configuration.DatabaseType;
+import org.apache.ambari.server.controller.internal.ClusterStackVersionResourceProvider;
import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
import org.apache.ambari.server.controller.internal.CredentialResourceProvider;
import org.apache.ambari.server.controller.internal.HostComponentResourceProvider;
@@ -460,6 +461,7 @@ public class ControllerModule extends AbstractModule {
.implement(ResourceProvider.class, Names.named("credential"), CredentialResourceProvider.class)
.implement(ResourceProvider.class, Names.named("kerberosDescriptor"), KerberosDescriptorResourceProvider.class)
.implement(ResourceProvider.class, Names.named("upgrade"), UpgradeResourceProvider.class)
+ .implement(ResourceProvider.class, Names.named("clusterStackVersion"), ClusterStackVersionResourceProvider.class)
.build(ResourceProviderFactory.class));
install(new FactoryModuleBuilder().implement(
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index a7b9d80..29f8e2a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -227,7 +227,8 @@ class DeleteIdentityHandler {
calculateConfig(kerberosDescriptor, serviceNames()),
new HashMap<>(),
false,
- new HashMap<>());
+ new HashMap<>(),
+ false);
return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index bb360b5..20c5708 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -59,6 +59,10 @@ public interface KerberosHelper {
*/
String DIRECTIVE_COMPONENTS = "regenerate_components";
/**
+ * directive used to pass host list to regenerate keytabs on
+ */
+ String DIRECTIVE_IGNORE_CONFIGS = "ignore_config_updates";
+ /**
* directive used to indicate that the enable Kerberos operation should proceed even if the
* cluster's security type is not changing
*/
@@ -591,6 +595,7 @@ public interface KerberosHelper {
* values
* @param configurations a Map of configurations to use a replacements for variables
* in identity fields
+ * @param ignoreHeadless boolean value to specify if headless principals must not be processed
* @return an integer indicating the number of identities added to the data file
* @throws java.io.IOException if an error occurs while writing a record to the data file
*/
@@ -598,9 +603,8 @@ public interface KerberosHelper {
Collection<KerberosIdentityDescriptor> identities,
Collection<String> identityFilter, String hostname, String serviceName,
String componentName, Map<String, Map<String, String>> kerberosConfigurations,
- Map<String, Map<String, String>> configurations)
+ Map<String, Map<String, String>> configurations, boolean ignoreHeadless)
throws IOException;
-
/**
* Calculates the map of configurations relative to the cluster and host.
* <p/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 013a063..67b08fd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -265,10 +265,13 @@ public class KerberosHelperImpl implements KerberosHelper {
Set<String> hostFilter = parseHostFilter(requestProperties);
Map<String, Set<String>> serviceComponentFilter = parseComponentFilter(requestProperties);
+ boolean updateConfigurations = !requestProperties.containsKey(DIRECTIVE_IGNORE_CONFIGS)
+ || !"true".equalsIgnoreCase(requestProperties.get(DIRECTIVE_IGNORE_CONFIGS));
+
if ("true".equalsIgnoreCase(value) || "all".equalsIgnoreCase(value)) {
- handler = new CreatePrincipalsAndKeytabsHandler(true, true, true);
+ handler = new CreatePrincipalsAndKeytabsHandler(true, updateConfigurations, true);
} else if ("missing".equalsIgnoreCase(value)) {
- handler = new CreatePrincipalsAndKeytabsHandler(false, true, true);
+ handler = new CreatePrincipalsAndKeytabsHandler(false, updateConfigurations, true);
}
if (handler != null) {
@@ -1482,7 +1485,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Collection<KerberosIdentityDescriptor> identities,
Collection<String> identityFilter, String hostname, String serviceName,
String componentName, Map<String, Map<String, String>> kerberosConfigurations,
- Map<String, Map<String, String>> configurations)
+ Map<String, Map<String, String>> configurations, boolean ignoreHeadless)
throws IOException {
int identitiesAdded = 0;
@@ -1534,7 +1537,8 @@ public class KerberosHelperImpl implements KerberosHelper {
keytabFileOwnerAccess,
keytabFileGroupName,
keytabFileGroupAccess,
- (keytabIsCachable) ? "true" : "false");
+ (keytabIsCachable) ? "true" : "false",
+ (ignoreHeadless && principalDescriptor.getType() == KerberosPrincipalType.USER) ? "true" : "false");
}
// Add the principal-related configuration to the map of configurations
@@ -2189,6 +2193,7 @@ public class KerberosHelperImpl implements KerberosHelper {
keytabFileOwnerAccess,
keytabFileGroupName,
keytabFileGroupAccess,
+ "false",
"false");
hostsWithValidKerberosClient.add(hostname);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
index 792a166..8c68f41 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RepositoryResponse.java
@@ -26,6 +26,8 @@ public class RepositoryResponse {
private String osType;
private String repoId;
private String repoName;
+ private String distribution;
+ private String components;
private String mirrorsList;
private String defaultBaseUrl;
private Long repositoryVersionId;
@@ -34,11 +36,14 @@ public class RepositoryResponse {
private boolean unique;
public RepositoryResponse(String baseUrl, String osType, String repoId,
- String repoName, String mirrorsList, String defaultBaseUrl) {
+ String repoName, String distribution, String components,
+ String mirrorsList, String defaultBaseUrl) {
setBaseUrl(baseUrl);
setOsType(osType);
setRepoId(repoId);
setRepoName(repoName);
+ setDistribution(distribution);
+ setComponents(components);
setMirrorsList(mirrorsList);
setDefaultBaseUrl(defaultBaseUrl);
}
@@ -97,6 +102,22 @@ public class RepositoryResponse {
this.repoName = repoName;
}
+ public String getDistribution() {
+ return distribution;
+ }
+
+ public void setDistribution(String distribution) {
+ this.distribution = distribution;
+ }
+
+ public String getComponents() {
+ return components;
+ }
+
+ public void setComponents(String components) {
+ this.components = components;
+ }
+
public String getMirrorsList() {
return mirrorsList;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java
index 3912138..0be7199 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java
@@ -22,6 +22,7 @@ package org.apache.ambari.server.controller;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.controller.internal.ClusterStackVersionResourceProvider;
import org.apache.ambari.server.controller.internal.UpgradeResourceProvider;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -68,4 +69,7 @@ public interface ResourceProviderFactory {
@Named("upgrade")
UpgradeResourceProvider getUpgradeResourceProvider(AmbariManagementController managementController);
+ @Named("clusterStackVersion")
+ ClusterStackVersionResourceProvider getClusterStackVersionResourceProvider(AmbariManagementController managementController);
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java
index b35b2a8..b4b13eb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java
@@ -183,7 +183,7 @@ public abstract class AbstractControllerResourceProvider extends AbstractAuthori
case StackVersion:
return new StackVersionResourceProvider(propertyIds, keyPropertyIds, managementController);
case ClusterStackVersion:
- return new ClusterStackVersionResourceProvider(managementController);
+ return resourceProviderFactory.getClusterStackVersionResourceProvider(managementController);
case HostStackVersion:
return new HostStackVersionResourceProvider(managementController);
case StackService:
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
index 82ff972..1cd2d10 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java
@@ -86,10 +86,6 @@ public abstract class AbstractProviderModule implements ProviderModule,
private static final int PROPERTY_REQUEST_CONNECT_TIMEOUT = 5000;
private static final int PROPERTY_REQUEST_READ_TIMEOUT = 10000;
- private static final String CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Clusters", "cluster_name");
- private static final String HOST_COMPONENT_CLUSTER_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "cluster_name");
- private static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name");
- private static final String HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "component_name");
private static final String GANGLIA_SERVER = "GANGLIA_SERVER";
private static final String METRIC_SERVER = "METRICS_COLLECTOR";
private static final String PROPERTIES_CATEGORY = "properties";
@@ -222,6 +218,11 @@ public abstract class AbstractProviderModule implements ProviderModule,
*/
private final Map<Resource.Type, List<PropertyProvider>> propertyProviders = new HashMap<>();
+ /*
+ * TODO: Instantiation for the concrete impl of this class is not done through
+ * dependency injector (guice) so none of these field initialization
+ * are going to work unless refactoring is complete.
+ */
@Inject
AmbariManagementController managementController;
@@ -523,7 +524,6 @@ public abstract class AbstractProviderModule implements ProviderModule,
@Override
public String getPort(String clusterName, String componentName, String hostName, boolean httpsEnabled) throws SystemException {
- // Parent map need not be synchronized
ConcurrentMap<String, ConcurrentMap<String, String>> clusterJmxPorts;
// Still need double check to ensure single init
if (!jmxPortMap.containsKey(clusterName)) {
@@ -534,9 +534,7 @@ public abstract class AbstractProviderModule implements ProviderModule,
}
}
}
-
clusterJmxPorts = jmxPortMap.get(clusterName);
-
Service.Type service = componentServiceMap.get(componentName);
if (service != null) {
@@ -883,15 +881,14 @@ public abstract class AbstractProviderModule implements ProviderModule,
for (Cluster cluster : clusterMap.values()) {
String clusterName = cluster.getClusterName();
-
Map<String, String> hostComponentMap = clusterHostComponentMap.get(clusterName);
+
if (hostComponentMap == null) {
hostComponentMap = new HashMap<>();
clusterHostComponentMap.put(clusterName, hostComponentMap);
}
List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts();
-
if (!CollectionUtils.isEmpty(serviceComponentHosts)) {
for (ServiceComponentHost sch : serviceComponentHosts) {
String componentName = sch.getServiceComponentName();
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index b4e1027..5a6e2cc 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -1298,12 +1298,16 @@ public class BlueprintConfigurationProcessor {
* elements in this property
*/
private static String[] splitAndTrimStrings(String propertyName) {
- List<String> namesWithoutWhitespace = new LinkedList<>();
- for (String service : propertyName.split(",")) {
- namesWithoutWhitespace.add(service.trim());
- }
+ if(propertyName != null) {
+ List<String> namesWithoutWhitespace = new LinkedList<>();
+ for (String service : propertyName.split(",")) {
+ namesWithoutWhitespace.add(service.trim());
+ }
- return namesWithoutWhitespace.toArray(new String[namesWithoutWhitespace.size()]);
+ return namesWithoutWhitespace.toArray(new String[namesWithoutWhitespace.size()]);
+ } else {
+ return new String[0];
+ }
}
/**
@@ -2726,7 +2730,7 @@ public class BlueprintConfigurationProcessor {
atlasPropsMap.put("atlas.kafka.bootstrap.servers", new MultipleHostTopologyUpdater("KAFKA_BROKER"));
atlasPropsMap.put("atlas.kafka.zookeeper.connect", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
atlasPropsMap.put("atlas.graph.index.search.solr.zookeeper-url", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER", ',', false, true, true));
- atlasPropsMap.put("atlas.graph.storage.hostname", new MultipleHostTopologyUpdater("HBASE_MASTER"));
+ atlasPropsMap.put("atlas.graph.storage.hostname", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
atlasPropsMap.put("atlas.audit.hbase.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER"));
// RANGER_ADMIN
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 0ad967b..ab1f05c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -445,6 +445,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
jsonContent.put("hostname", hostName);
jsonContent.put("public_hostname", publicHostName);
jsonContent.put("clusterName", cluster.getClusterName());
+ jsonContent.put("serviceName", serviceName);
+ jsonContent.put("role", componentName);
jsonConfigurations = gson.toJson(jsonContent);
File tmpDirectory = new File(TMP_PATH);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
index 4d5a4ac..ba5a4e7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java
@@ -41,13 +41,11 @@ import org.apache.ambari.server.actionmanager.ActionManager;
import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.actionmanager.StageFactory;
-import org.apache.ambari.server.agent.stomp.MetadataHolder;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.ActionExecutionContext;
import org.apache.ambari.server.controller.AmbariActionExecutionHelper;
import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
import org.apache.ambari.server.controller.RequestStatusResponse;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
@@ -205,12 +203,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
@Inject
private static Provider<Clusters> clusters;
- @Inject
- private static Provider<MetadataHolder> m_metadataHolder;
-
- @Inject
- private static Provider<AmbariManagementControllerImpl> m_ambariManagementController;
-
/**
* Used for updating the existing stack tools with those of the stack being
* distributed.
@@ -221,6 +213,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
/**
* Constructor.
*/
+ @Inject
public ClusterStackVersionResourceProvider(
AmbariManagementController managementController) {
super(propertyIds, keyPropertyIds, managementController);
@@ -462,7 +455,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
}
- @Transactional
+ @Transactional(rollbackOn = {RuntimeException.class, SystemException.class, AmbariException.class})
RequestStatus createOrUpdateHostVersions(Cluster cluster,
RepositoryVersionEntity repoVersionEntity, VersionDefinitionXml versionDefinitionXml,
StackId stackId, boolean forceInstalled, Map<String, Object> propertyMap)
@@ -505,6 +498,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
}
+ checkPatchVDFAvailableServices(cluster, repoVersionEntity, versionDefinitionXml);
// the cluster will create/update all of the host versions to the correct state
List<Host> hostsNeedingInstallCommands = cluster.transitionHostsToInstalling(
@@ -603,9 +597,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
// determine services for the repo
Set<String> serviceNames = new HashSet<>();
-
- checkPatchVDFAvailableServices(cluster, repoVersionEnt, desiredVersionDefinition);
-
// !!! limit the serviceNames to those that are detailed for the repository.
// TODO packages don't have component granularity
if (RepositoryType.STANDARD != repoVersionEnt.getType()) {
@@ -676,11 +667,13 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
}
}
- private ActionExecutionContext getHostVersionInstallCommand(RepositoryVersionEntity repoVersion,
+ @Transactional
+ ActionExecutionContext getHostVersionInstallCommand(RepositoryVersionEntity repoVersion,
Cluster cluster, AmbariManagementController managementController, AmbariMetaInfo ami,
final StackId stackId, Set<String> repoServices, Stage stage1, Host host)
throws SystemException {
+
// Determine repositories for host
String osFamily = host.getOsFamily();
@@ -694,7 +687,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
if (null == osEntity || CollectionUtils.isEmpty(osEntity.getRepositories())) {
throw new SystemException(String.format("Repositories for os type %s are " +
- "not defined. Repo version=%s, stackId=%s",
+ "not defined for version %s of Stack %s.",
osFamily, repoVersion.getVersion(), stackId));
}