You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/11/12 20:35:00 UTC
ambari git commit: AMBARI-13824. Stop-and-Start Upgrade: Fix for
Oozie Express upgrade issues from HDP 2.1 to 2.3 (Swapan Shridhar via
alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 8429a6a1d -> 7f725853f
AMBARI-13824. Stop-and-Start Upgrade: Fix for Oozie Express upgrade issues from HDP 2.1 to 2.3 (Swapan Shridhar via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7f725853
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7f725853
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7f725853
Branch: refs/heads/trunk
Commit: 7f725853f6e4282ef0710371c57d18d784998527
Parents: 8429a6a
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Nov 12 11:33:09 2015 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Nov 12 11:34:51 2015 -0800
----------------------------------------------------------------------
.../libraries/functions/conf_select.py | 31 +++++++++----
.../AmbariCustomCommandExecutionHelper.java | 37 +++++++++++++--
.../internal/UpgradeResourceProvider.java | 47 ++++++++++----------
.../state/stack/upgrade/ConfigureFunction.java | 36 +++++++++++++++
.../server/state/stack/upgrade/Grouping.java | 3 ++
.../state/stack/upgrade/StageWrapper.java | 7 +--
.../ambari/server/state/stack/upgrade/Task.java | 8 +++-
.../4.0.0.2.0/package/scripts/oozie_server.py | 20 +++++++--
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 7 ++-
9 files changed, 153 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index 620f021..9bf259b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -224,14 +224,12 @@ def create(stack_name, package, version, dry_run = False):
def select(stack_name, package, version, try_create=True):
"""
- Selects a config version for the specified package. Currently only works if the version is
- for HDP-2.3 or higher
+ Selects a config version for the specified package.
:stack_name: the name of the stack
:package: the name of the package, as-used by conf-select
:version: the version number to create
:try_create: optional argument to attempt to create the directory before setting it
"""
-
if not _valid(stack_name, package, version):
return
@@ -240,6 +238,28 @@ def select(stack_name, package, version, try_create=True):
shell.checked_call(get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False, sudo=True)
+ # Create the symbolic link using 'PACKAGE_DIRS' for the given package
+ # Starting with 2.3, we have sym links instead of flat directories.
+ # Eg: /etc/<service-name>/conf -> /etc/<service-name>/2.3.x.y-<version>/0
+ # But, in case of Express upgrade from HDP 2.1-> HDP 2.3 where we have
+ # deleted the /etc/<service-name>/conf directory, the above mentioned
+ # symlink needs to be created here.
+ if package in PACKAGE_DIRS:
+ conf_dirs = PACKAGE_DIRS[package]
+ Logger.info("For package : {0}, DIRS = {1}".format(package, conf_dirs))
+ for dirInfo in conf_dirs:
+ if "conf_dir" in dirInfo and "current_dir" in dirInfo:
+ conf_dir = dirInfo["conf_dir"]
+ current_dir = dirInfo["current_dir"]
+ Logger.info("For package : {0}, Source dir: {1}, Dest dir: {2}".format(package, conf_dir, current_dir))
+ if os.path.exists(current_dir):
+ real_path_of_current_dir = os.path.realpath(current_dir)
+ normalized_conf_dir = (os.path.normpath(conf_dir)).strip()
+ normalized_current_dir = (os.path.normpath(real_path_of_current_dir)).strip()
+ Logger.info("Normalized Conf Dir : {0}, Normalized Current Dir : {1}".format(normalized_conf_dir, normalized_current_dir))
+ if not os.path.isdir(normalized_conf_dir) and os.path.isdir(normalized_current_dir) and normalized_current_dir != normalized_conf_dir:
+ Logger.info("Creating Symlink : {0} -> {1}".format(normalized_conf_dir, normalized_current_dir))
+ os.symlink(normalized_current_dir, normalized_conf_dir)
def get_hadoop_conf_dir(force_latest_on_upgrade=False):
"""
@@ -293,23 +313,18 @@ def create_config_links(stack_id, stack_version):
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
-
if stack_id is None:
Logger.info("Cannot create config links when stack_id is not defined")
return
-
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info("Unrecognized stack id {0}".format(stack_id))
return
-
if args[0] != "HDP":
Logger.info("Unrecognized stack name {0}".format(args[0]))
-
if version.compare_versions(version.format_hdp_stack_version(args[1]), "2.3.0.0") < 0:
Logger.info("Cannot link configs unless HDP-2.3 or higher")
return
-
for k, v in PACKAGE_DIRS.iteritems():
dirs = create(args[0], k, stack_version, dry_run = True)
if 0 == len(dirs):
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 9ba5a22..1f8c687 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -180,10 +180,41 @@ public class AmbariCustomCommandExecutionHelper {
stackId.getStackName(), stackId.getStackVersion(),
serviceName, componentName);
- return !(!componentInfo.isCustomCommand(commandName) &&
- !actionMetadata.isDefaultHostComponentCommand(commandName));
+ return actionMetadata.isDefaultHostComponentCommand(commandName);
}
+ private Boolean isValidDefaultCommand(String clusterName,
+ String serviceName, String componentName, String commandName)
+ throws AmbariException {
+
+ Cluster cluster = clusters.getCluster(clusterName);
+ StackId stackId = cluster.getDesiredStackVersion();
+
+ if (componentName == null) {
+ return false;
+ }
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+ stackId.getStackName(), stackId.getStackVersion(),
+ serviceName, componentName);
+
+ return (actionMetadata.isDefaultHostComponentCommand(commandName));
+ }
+
+ private Boolean isValidDefaultCommand(ActionExecutionContext
+ actionExecutionContext, RequestResourceFilter resourceFilter)
+ throws AmbariException {
+ String clusterName = actionExecutionContext.getClusterName();
+ String serviceName = resourceFilter.getServiceName();
+ String componentName = resourceFilter.getComponentName();
+ String commandName = actionExecutionContext.getActionName();
+
+ if (componentName == null) {
+ return false;
+ }
+
+ return isValidCustomCommand(clusterName, serviceName, componentName, commandName);
+ }
+
private Boolean isValidCustomCommand(ActionExecutionContext
actionExecutionContext, RequestResourceFilter resourceFilter)
throws AmbariException {
@@ -908,7 +939,7 @@ public class AmbariCustomCommandExecutionHelper {
findHostAndAddServiceCheckAction(actionExecutionContext, resourceFilter, stage);
} else if (actionName.equals(DECOMMISSION_COMMAND_NAME)) {
addDecommissionAction(actionExecutionContext, resourceFilter, stage);
- } else if (isValidCustomCommand(actionExecutionContext, resourceFilter)) {
+ } else if (isValidDefaultCommand(actionExecutionContext, resourceFilter)) {
String commandDetail = getReadableCustomCommandDetail(actionExecutionContext, resourceFilter);
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 8dff73a..748dbbe 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -17,25 +17,10 @@
*/
package org.apache.ambari.server.controller.internal;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
-
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
@@ -108,10 +93,24 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
/**
* Manages the ability to start and get status of upgrades.
@@ -1095,6 +1094,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
throws AmbariException {
switch (wrapper.getType()) {
+ case CONFIGURE:
case START:
case STOP:
case RESTART:
@@ -1242,6 +1242,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
String function = null;
switch (wrapper.getType()) {
+ case CONFIGURE:
case START:
case STOP:
case RESTART:
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
new file mode 100644
index 0000000..fc6a45b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
@@ -0,0 +1,36 @@
+package org.apache.ambari.server.state.stack.upgrade;
+
+ import javax.xml.bind.annotation.XmlAccessType;
+ import javax.xml.bind.annotation.XmlAccessorType;
+ import javax.xml.bind.annotation.XmlRootElement;
+ import javax.xml.bind.annotation.XmlTransient;
+ import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Used to represent Configuring of a component.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name="configure_function")
+public class ConfigureFunction extends Task {
+
+ @XmlTransient
+ private Task.Type type = Type.CONFIGURE_FUNCTION;
+
+ public static final String actionVerb = "Configuring";
+
+ @Override
+ public Task.Type getType() {
+ return type;
+ }
+
+ @Override
+ public StageWrapper.Type getStageWrapperType() {
+ return StageWrapper.Type.CONFIGURE;
+ }
+
+ @Override
+ public String getActionVerb() {
+ return actionVerb;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index 6122e2d..04e9aaa 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -244,6 +244,9 @@ public class Grouping {
case EXECUTE:
type = StageWrapper.Type.RU_TASKS;
break;
+ case CONFIGURE_FUNCTION:
+ type = StageWrapper.Type.CONFIGURE;
+ break;
case RESTART:
type = StageWrapper.Type.RESTART;
break;
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
index 2ea3671..86a8f55 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.state.stack.upgrade;
+import com.google.gson.Gson;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -25,8 +27,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.gson.Gson;
-
/**
*
*/
@@ -149,6 +149,7 @@ public class StageWrapper {
RU_TASKS,
SERVICE_CHECK,
STOP,
- START
+ START,
+ CONFIGURE
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
index f443e53..60091a6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
@@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
/**
* Base class to identify the items that could possibly occur during an upgrade
*/
-@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class, StartTask.class, StopTask.class, ServerActionTask.class})
+@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class, StartTask.class, StopTask.class, ServerActionTask.class, ConfigureFunction.class})
public abstract class Task {
/**
@@ -66,6 +66,10 @@ public abstract class Task {
*/
CONFIGURE,
/**
+ * Task that sets up the configuration for subsequent task
+ */
+ CONFIGURE_FUNCTION,
+ /**
* Task that displays a message and must be confirmed before continuing
*/
MANUAL,
@@ -101,7 +105,7 @@ public abstract class Task {
* @return {@code true} if the task is a command type (as opposed to an action)
*/
public boolean isCommand() {
- return this == RESTART || this == START || this == STOP || this == SERVICE_CHECK;
+ return this == RESTART || this == START || this == CONFIGURE_FUNCTION || this == STOP || this == SERVICE_CHECK;
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
index f35df93..35975df 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
@@ -24,6 +24,7 @@ from resource_management.libraries.functions import compare_versions
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import format_hdp_stack_version
+from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.functions.security_commons import build_expectations
from resource_management.libraries.functions.security_commons import cached_kinit_executor
from resource_management.libraries.functions.security_commons import get_params_from_filesystem
@@ -47,8 +48,20 @@ class OozieServer(Script):
def install(self, env):
self.install_packages(env)
- def configure(self, env):
+ def configure(self, env, upgrade_type=None):
import params
+
+ if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \
+ params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ conf_select.select(params.stack_name, "oozie", params.version)
+ # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of
+ # oozie, we need to create the symlinks both for server and client.
+ # This is required as both need to be pointing to new installed oozie version.
+
+ # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
+ hdp_select.select("oozie-client", params.version)
+ # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
+ hdp_select.select("oozie-server", params.version)
env.set_params(params)
oozie(is_server=True)
@@ -166,8 +179,9 @@ class OozieServerDefault(OozieServer):
OozieUpgrade.backup_configuration()
- conf_select.select(params.stack_name, "oozie", params.version)
- hdp_select.select("oozie-server", params.version)
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ conf_select.select(params.stack_name, "oozie", params.version)
+ hdp_select.select("oozie-server", params.version)
OozieUpgrade.restore_configuration()
OozieUpgrade.prepare_libext_directory()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7f725853/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index f12d0cd..49c8385 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -697,6 +697,11 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
+ <!-- We need to set up the "/etc/oozie/conf" symlink before upgrading, as the HDP 2.1 directory
+ pertaining to oozie has been deleted as part of HDP 2.1 removal in Express Upgrade
+ from HDP 2.1->2.3.
+ -->
+ <task xsi:type="configure_function"/>
<task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
@@ -775,4 +780,4 @@
</component>
</service>
</processing>
-</upgrade>
\ No newline at end of file
+</upgrade>