You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/10 01:16:53 UTC
[6/9] incubator-slider git commit: SLIDER-623 make python checks
optional for all but the AM tests that spawn workers
SLIDER-623 make python checks optional for all but the AM tests that spawn workers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/79452935
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/79452935
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/79452935
Branch: refs/heads/releases/slider-0.60
Commit: 79452935fe60ce451ce4ad3299a1c4e71964516f
Parents: 4c7d83f
Author: Steve Loughran <st...@apache.org>
Authored: Sun Nov 9 19:07:48 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 10 00:15:47 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/common/SliderXmlConfKeys.java | 7 +++++++
.../org/apache/slider/common/tools/SliderUtils.java | 12 ++++++++----
.../apache/slider/server/appmaster/SliderAppMaster.java | 7 ++++---
.../slider/common/tools/TestExecutionEnvironment.groovy | 8 ++++++--
.../apache/slider/providers/agent/AgentTestBase.groovy | 10 ++++++----
.../apache/slider/providers/agent/TestAgentEcho.groovy | 5 +++++
.../apache/slider/test/YarnMiniClusterTestBase.groovy | 6 ++++--
7 files changed, 40 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 1c90e3b..e7f8ce5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -149,4 +149,11 @@ public interface SliderXmlConfKeys {
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
String KEY_SECURITY_ENABLED = "site.global.security_enabled";
+
+ /**
+ * Set to disable server-side checks for python, openssl &c.
+ * This should only be set for testing
+ */
+ String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
+ "slider.am.dependency.checks.disabled";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 247c25d..db398a6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1975,15 +1975,19 @@ public final class SliderUtils {
* This looks for everything felt to be critical for execution, including
* native binaries and other essential dependencies.
* @param logger logger to log to on normal execution
+ * @param dependencyChecks flag to indicate checks for agent dependencies
* @throws IOException on IO failures
* @throws SliderException on validation failures
*/
- public static void validateSliderServerEnvironment(Logger logger) throws
+ public static void validateSliderServerEnvironment(Logger logger,
+ boolean dependencyChecks) throws
IOException,
SliderException {
- maybeVerifyWinUtilsValid(logger);
- validatePythonEnv(logger);
- validateOpenSSLEnv(logger);
+ maybeVerifyWinUtilsValid();
+ if (dependencyChecks) {
+ validatePythonEnv(logger);
+ validateOpenSSLEnv(logger);
+ }
}
public static void validateOpenSSLEnv(Logger logger) throws
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8791d19..7000583 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -106,7 +105,6 @@ import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
@@ -438,7 +436,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
checkAndWarnForAuthTokenProblems();
// validate server env
- SliderUtils.validateSliderServerEnvironment(log);
+ boolean dependencyChecks =
+ !conf.getBoolean(KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ false);
+ SliderUtils.validateSliderServerEnvironment(log, dependencyChecks);
executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
Executors.newFixedThreadPool(2,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
index 97b72d1..3b2f992 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -35,12 +35,16 @@ class TestExecutionEnvironment extends SliderTestBase {
@Test
public void testWinutils() throws Throwable {
SliderUtils.maybeVerifyWinUtilsValid();
-
}
@Test
public void testServerEnv() throws Throwable {
- SliderUtils.validateSliderServerEnvironment(log)
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ }
+
+ @Test
+ public void testServerEnvNoDependencies() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log, false)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index ea10494..5bf1c1f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -30,7 +30,6 @@ import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
-import org.junit.BeforeClass
import org.junit.Rule
import org.junit.rules.TemporaryFolder
@@ -52,9 +51,12 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
/**
* Server side test: validate system env before launch
*/
- @BeforeClass
- public static void checkServerEnv() {
-// SliderUtils.validateSliderServerEnvironment(null)
+ public static void assumeValidServerEnv() {
+ try {
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ } catch (Exception e) {
+ skip(e.toString())
+ }
}
public String app_def_pkg_path;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index f40d5a7..2eb39e3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Before
@@ -68,6 +69,8 @@ class TestAgentEcho extends AgentTestBase {
@Test
public void testEchoOperation() throws Throwable {
+ assumeValidServerEnv()
+
String clustername = createMiniCluster("",
configuration,
1,
@@ -95,6 +98,8 @@ class TestAgentEcho extends AgentTestBase {
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ARG_DEFINE,
+ SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false"
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/79452935/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 4d2cb3b..73b81ab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -88,7 +88,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false)
+ SLIDER_CONFIG.setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ true)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1)
+
}
@@ -345,9 +348,8 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
public YarnConfiguration getTestConfiguration() {
YarnConfiguration conf = getConfiguration()
-
conf.addResource(SLIDER_TEST_XML)
- return conf
+ return conf;
}
protected String getRMAddr() {