You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/09 20:07:58 UTC
[1/2] incubator-slider git commit: SLIDER-622 move test-side native
lib checks into SliderUtils for AM to use too
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-623-python f282efd5a -> 4d3dca8c0
SLIDER-622 move test-side native lib checks into SliderUtils for AM to use too
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c566fbe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c566fbe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c566fbe8
Branch: refs/heads/feature/SLIDER-623-python
Commit: c566fbe82ad5d19a3e70e65b5cfb802c08e801f8
Parents: f282efd
Author: Steve Loughran <st...@apache.org>
Authored: Sun Nov 9 18:53:36 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 9 18:54:31 2014 +0000
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 52 ++++++++++++++------
.../tools/TestExecutionEnvironment.groovy | 2 +-
.../common/tools/TestWindowsSupport.groovy | 2 +-
.../apache/slider/test/SliderTestUtils.groovy | 38 ++------------
4 files changed, 41 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c566fbe8/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 1a9b8fb..247c25d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1720,6 +1721,36 @@ public final class SliderUtils {
return is;
}
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath("winutils.exe");
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No WINUTILS.EXE. ");
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
/**
* Strictly verify that windows utils is present.
@@ -1727,25 +1758,14 @@ public final class SliderUtils {
* the headers.
* @throws IOException on any problem reading the file
* @throws FileNotFoundException if the file is not considered valid
- * @param logger
*/
- public static void maybeVerifyWinUtilsValid(Logger logger) throws
+ public static void maybeVerifyWinUtilsValid() throws
IOException,
SliderException {
- if (!Shell.WINDOWS) {
- return;
- }
- String exePath = Shell.getWinUtilsPath();
- String program = WINUTILS;
- if (exePath == null) {
- throw new FileNotFoundException(program + " not found on Path : " +
- System.getenv("Path"));
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
}
- File exe = new File(exePath);
-
- verifyWindowsExe(program, exe);
- execCommand(WINUTILS, 0, 5000, log, null, exePath, "systeminfo");
-
}
public static void verifyIsFile(String program, File exe) throws
@@ -1947,7 +1967,7 @@ public final class SliderUtils {
public static void validateSliderClientEnvironment(Logger logger) throws
IOException,
SliderException {
- maybeVerifyWinUtilsValid(logger);
+ maybeVerifyWinUtilsValid();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c566fbe8/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
index ad78c0e..97b72d1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -34,7 +34,7 @@ class TestExecutionEnvironment extends SliderTestBase {
@Test
public void testWinutils() throws Throwable {
- SliderUtils.maybeVerifyWinUtilsValid(log);
+ SliderUtils.maybeVerifyWinUtilsValid();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c566fbe8/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 61960e6..6351c14 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -134,7 +134,7 @@ class TestWindowsSupport extends SliderTestBase {
@Test
public void testHasWinutils() throws Throwable {
assume(Shell.WINDOWS, "not windows")
- SliderUtils.maybeVerifyWinUtilsValid(log)
+ SliderUtils.maybeVerifyWinUtilsValid()
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c566fbe8/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 6e63da9..87621a1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -27,9 +27,7 @@ import org.apache.commons.httpclient.methods.GetMethod
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem as HadoopFS
-import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.nativeio.NativeIO
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -191,45 +189,15 @@ class SliderTestUtils extends Assert {
fail("Not implemented")
}
- /**
- * Check for any needed libraries being present. On Unix none are needed;
- * on windows they must be present
- * @return true if all is well
- */
- public static String checkForRequiredLibraries() {
-
- if (!Shell.WINDOWS) {
- return "";
- }
- StringBuilder errorText = new StringBuilder("")
- boolean available = true;
- if (!NativeIO.available) {
- errorText.append("No native IO library. ")
- }
- try {
- def path = Shell.getQualifiedBinPath("winutils.exe");
- log.debug("winutils is at $path")
- } catch (IOException e) {
- errorText.append("No WINUTILS.EXE. ")
- log.warn("No winutils: $e", e)
- }
- try {
- File target = new File("target")
- FileUtil.canRead(target)
- } catch (UnsatisfiedLinkError e) {
- log.warn("Failing to link to native IO methods: $e", e)
- errorText.append("No native IO methods")
- }
- return errorText.toString();
- }
+
/**
* Assert that any needed libraries being present. On Unix none are needed;
* on windows they must be present
*/
public static void assertNativeLibrariesPresent() {
- String errorText = checkForRequiredLibraries()
- if (errorText != null) {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries()
+ if (errorText != "") {
fail(errorText)
}
}
[2/2] incubator-slider git commit: SLIDER-623 make python checks
optional for all but the AM tests that spawn workers
Posted by st...@apache.org.
SLIDER-623 make python checks optional for all but the AM tests that spawn workers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4d3dca8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4d3dca8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4d3dca8c
Branch: refs/heads/feature/SLIDER-623-python
Commit: 4d3dca8c094595b9c088536b82ff8ddecd7cc748
Parents: c566fbe
Author: Steve Loughran <st...@apache.org>
Authored: Sun Nov 9 19:07:48 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 9 19:07:48 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/common/SliderXmlConfKeys.java | 7 +++++++
.../org/apache/slider/common/tools/SliderUtils.java | 12 ++++++++----
.../apache/slider/server/appmaster/SliderAppMaster.java | 7 ++++---
.../slider/common/tools/TestExecutionEnvironment.groovy | 8 ++++++--
.../apache/slider/providers/agent/AgentTestBase.groovy | 10 ++++++----
.../apache/slider/providers/agent/TestAgentEcho.groovy | 5 +++++
.../apache/slider/test/YarnMiniClusterTestBase.groovy | 6 ++++--
7 files changed, 40 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 1c90e3b..e7f8ce5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -149,4 +149,11 @@ public interface SliderXmlConfKeys {
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
String KEY_SECURITY_ENABLED = "site.global.security_enabled";
+
+ /**
+ * Set to disable server-side checks for python, openssl &c.
+ * This should only be set for testing
+ */
+ String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
+ "slider.am.dependency.checks.disabled";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 247c25d..db398a6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1975,15 +1975,19 @@ public final class SliderUtils {
* This looks for everything felt to be critical for execution, including
* native binaries and other essential dependencies.
* @param logger logger to log to on normal execution
+ * @param dependencyChecks flag to indicate checks for agent dependencies
* @throws IOException on IO failures
* @throws SliderException on validation failures
*/
- public static void validateSliderServerEnvironment(Logger logger) throws
+ public static void validateSliderServerEnvironment(Logger logger,
+ boolean dependencyChecks) throws
IOException,
SliderException {
- maybeVerifyWinUtilsValid(logger);
- validatePythonEnv(logger);
- validateOpenSSLEnv(logger);
+ maybeVerifyWinUtilsValid();
+ if (dependencyChecks) {
+ validatePythonEnv(logger);
+ validateOpenSSLEnv(logger);
+ }
}
public static void validateOpenSSLEnv(Logger logger) throws
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8791d19..7000583 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -106,7 +105,6 @@ import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
@@ -438,7 +436,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
checkAndWarnForAuthTokenProblems();
// validate server env
- SliderUtils.validateSliderServerEnvironment(log);
+ boolean dependencyChecks =
+ !conf.getBoolean(KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ false);
+ SliderUtils.validateSliderServerEnvironment(log, dependencyChecks);
executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
Executors.newFixedThreadPool(2,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
index 97b72d1..3b2f992 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -35,12 +35,16 @@ class TestExecutionEnvironment extends SliderTestBase {
@Test
public void testWinutils() throws Throwable {
SliderUtils.maybeVerifyWinUtilsValid();
-
}
@Test
public void testServerEnv() throws Throwable {
- SliderUtils.validateSliderServerEnvironment(log)
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ }
+
+ @Test
+ public void testServerEnvNoDependencies() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log, false)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index ea10494..5bf1c1f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -30,7 +30,6 @@ import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
-import org.junit.BeforeClass
import org.junit.Rule
import org.junit.rules.TemporaryFolder
@@ -52,9 +51,12 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
/**
* Server side test: validate system env before launch
*/
- @BeforeClass
- public static void checkServerEnv() {
-// SliderUtils.validateSliderServerEnvironment(null)
+ public static void assumeValidServerEnv() {
+ try {
+ SliderUtils.validateSliderServerEnvironment(log, true)
+ } catch (Exception e) {
+ skip(e.toString())
+ }
}
public String app_def_pkg_path;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index f40d5a7..2eb39e3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Before
@@ -68,6 +69,8 @@ class TestAgentEcho extends AgentTestBase {
@Test
public void testEchoOperation() throws Throwable {
+ assumeValidServerEnv()
+
String clustername = createMiniCluster("",
configuration,
1,
@@ -95,6 +98,8 @@ class TestAgentEcho extends AgentTestBase {
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ARG_DEFINE,
+ SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false"
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4d3dca8c/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 4d2cb3b..73b81ab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -88,7 +88,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false)
SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false)
+ SLIDER_CONFIG.setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED,
+ true)
SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1)
+
}
@@ -345,9 +348,8 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
public YarnConfiguration getTestConfiguration() {
YarnConfiguration conf = getConfiguration()
-
conf.addResource(SLIDER_TEST_XML)
- return conf
+ return conf;
}
protected String getRMAddr() {