You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/06/25 03:11:57 UTC
[1/6] git commit: SLIDER-160: initial client settings for an ambari
vagrant VM up on c6401
Repository: incubator-slider
Updated Branches:
refs/heads/develop 3be6c8692 -> 186673363
SLIDER-160: initial client settings for an ambari vagrant VM up on c6401
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/81578df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/81578df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/81578df2
Branch: refs/heads/develop
Commit: 81578df22f2eee22f49419e48a2f3e07c33005ff
Parents: 153f5e9
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 17:35:23 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 17:35:23 2014 -0700
----------------------------------------------------------------------
src/test/clusters/c6401/slider/log4j.properties | 83 ++++++++++++++++++++
.../clusters/c6401/slider/slider-client.xml | 71 +++++++++++++++++
2 files changed, 154 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/81578df2/src/test/clusters/c6401/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/log4j.properties b/src/test/clusters/c6401/slider/log4j.properties
new file mode 100644
index 0000000..6211771
--- /dev/null
+++ b/src/test/clusters/c6401/slider/log4j.properties
@@ -0,0 +1,83 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/81578df2/src/test/clusters/c6401/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/slider-client.xml b/src/test/clusters/c6401/slider/slider-client.xml
new file mode 100644
index 0000000..f1aa4fb
--- /dev/null
+++ b/src/test/clusters/c6401/slider/slider-client.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM
+-->
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>configs/c6401/slider</value>
+ <description>This is just for diagnostics</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>c6401:8050</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://c6401.ambari.apache.org:8020</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>c6401:2181</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+ /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ </value>
+ </property>
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value>60000</value>
+ </property>
+
+
+</configuration>
[5/6] git commit: SLIDER-153 remove superflous check for agent test
enablement in TestAppsThroughAgent
Posted by st...@apache.org.
SLIDER-153 remove superflous check for agent test enablement in TestAppsThroughAgent
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bab88672
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bab88672
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bab88672
Branch: refs/heads/develop
Commit: bab886723eca65bbe3a4090642b26714391df3ba
Parents: eba97da
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 18:10:02 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 18:10:02 2014 -0700
----------------------------------------------------------------------
.../apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bab88672/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
index e3422ee..793a323 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
@@ -43,10 +43,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testCreateFlex() throws Throwable {
- if (!AGENTTESTS_ENABLED) {
- log.info "TESTS are not run."
- return
- }
+ assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
SliderShell shell = slider(EXIT_SUCCESS,
[6/6] git commit: Merge branch
'feature/SLIDER-160_Improve_slider_functional_testing' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-160_Improve_slider_functional_testing' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/18667336
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/18667336
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/18667336
Branch: refs/heads/develop
Commit: 186673363c32a644d5ebaa21e3e2f7e3150115d6
Parents: 3be6c86 bab8867
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 18:10:28 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 18:10:28 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 68 +++++++++++++
.../funtest/framework/CommandTestBase.groovy | 9 +-
.../funtest/framework/FileUploader.groovy | 100 +++++++++++++++++++
.../funtest/framework/FuntestProperties.groovy | 12 ++-
.../slider/funtest/framework/SudoClosure.groovy | 54 ++++++++++
.../lifecycle/AgentCommandTestBase.groovy | 39 +++-----
.../lifecycle/TestAppsThroughAgent.groovy | 5 +-
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
src/test/clusters/c6401/slider/log4j.properties | 83 +++++++++++++++
.../clusters/c6401/slider/slider-client.xml | 71 +++++++++++++
10 files changed, 403 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
[3/6] git commit: SLIDER-160: agent functional tests upload
binaries/ini file if there is a difference
Posted by st...@apache.org.
SLIDER-160: agent functional tests upload binaries/ini file if there is a difference
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5464584d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5464584d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5464584d
Branch: refs/heads/develop
Commit: 5464584d784cf57348bf22100cee2581d5a887c0
Parents: 8327e52
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 17:40:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 17:40:49 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 41 +++++++++++++++---
.../funtest/framework/CommandTestBase.groovy | 9 ++--
.../funtest/framework/FileUploader.groovy | 45 ++++++++++++++------
.../funtest/framework/FuntestProperties.groovy | 12 ++++--
.../lifecycle/AgentCommandTestBase.groovy | 39 ++++++-----------
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
6 files changed, 94 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index b1c29c0..5f10c0e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -20,22 +20,51 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
-class AgentUploads {
+class AgentUploads implements FuntestProperties {
final Configuration conf
+ private final FileUploader uploader
+ private final HadoopFS clusterFS
+ private final Path homeDir
+
AgentUploads(Configuration conf) {
this.conf = conf
+ uploader = new FileUploader(conf, UserGroupInformation.currentUser)
+ clusterFS = uploader.fileSystem
+ homeDir = clusterFS.homeDirectory
}
- def execUploadSequence() {
- UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn")
+ /**
+ * Upload agent-related files
+ * @param tarballDir
+ * @param force
+ * @return
+ */
+ def uploadAgentFiles(File tarballDir, boolean force) {
+ def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR)
+ assert localAgentTar.exists()
+
+ def agentTarballPath = new Path(
+ homeDir,
+ AGENT_TAR_FILENAME)
+
+ // Upload the agent tarball
+ uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force)
+
+ File localAgentIni = new File(tarballDir, AGENT_INI_IN_SLIDER_TAR)
+ // Upload the agent.ini
+ def agentIniPath = new Path(homeDir, AGENT_INI)
+ uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force)
+
+ return [agentTarballPath, agentIniPath]
+
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d2accbb..61ae804 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -50,17 +50,16 @@ abstract class CommandTestBase extends SliderTestUtils {
LoggerFactory.getLogger(CommandTestBase.class);
public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP)
- public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP)
- public static final File SLIDER_BIN_DIRECTORY = new File(
- SLIDER_BIN_DIR).canonicalFile
+ public static final String SLIDER_TAR_DIR = sysprop(SLIDER_BIN_DIR_PROP)
+ public static final File SLIDER_TAR_DIRECTORY = new File(
+ SLIDER_TAR_DIR).canonicalFile
public static final File SLIDER_SCRIPT = new File(
- SLIDER_BIN_DIRECTORY,
+ SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
-
public static final YarnConfiguration SLIDER_CONFIG
public static final int THAW_WAIT_TIME
public static final int FREEZE_WAIT_TIME
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index dd42473..262693c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -23,8 +23,9 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.AccessControlException
import org.apache.hadoop.security.UserGroupInformation
-
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
class FileUploader {
final Configuration conf
@@ -38,39 +39,59 @@ class FileUploader {
/**
* Copy if the file is considered out of date
* @param src
- * @param dest
+ * @param destPath
* @param force
* @return
*/
- public boolean copyIfOutOfDate(File src, Path dest, boolean force) {
+ public boolean copyIfOutOfDate(File src, Path destPath, boolean force) {
def srcLen = src.length()
- def fs = getFileSystem(user, dest.toUri())
+ def fs = getFileSystem(destPath)
boolean toCopy = force
if (!toCopy) {
try {
- def status = fs.getFileStatus(dest)
+ def status = fs.getFileStatus(destPath)
toCopy = status.len != srcLen
} catch (FileNotFoundException fnfe) {
toCopy = true;
}
}
if (toCopy) {
- log.info("Copying $src to $dest")
- fs.mkdirs(dest, FsPermission.dirDefault)
- return FileUtil.copy(src, fs, dest, false, conf)
+ log.info("Copying $src to $destPath")
+ try {
+ fs.delete(destPath, true)
+ fs.mkdirs(destPath.getParent(), FsPermission.dirDefault)
+ return FileUtil.copy(src, fs, destPath, false, conf)
+ } catch (AccessControlException ace) {
+ log.error("No write access to test user home directory. " +
+ "Ensure home directory exists and has correct permissions." +
+ ace, ace)
+ throw ace
+ }
} else {
- log.debug("Skipping copy as the destination $dest considered up to date")
+ log.debug("Skipping copy as the destination $destPath considered up to date")
return false;
}
+ }
+ public HadoopFS getFileSystem(Path dest) {
+ getFileSystem(user, dest)
+ }
+
+ public HadoopFS getFileSystem() {
+ getFileSystem(user, HadoopFS.getDefaultUri(conf))
}
- public static def getFileSystem(
+ public def getFileSystem(
+ UserGroupInformation user, final Path path) {
+ return getFileSystem(user, path.toUri())
+
+ }
+ public def getFileSystem(
UserGroupInformation user, final URI uri) {
+
SudoClosure.sudo(user) {
- org.apache.hadoop.fs.FileSystem.get(uri, conf);
+ HadoopFS.get(uri, conf);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 8cbc098..9b63c22 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,8 +57,12 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting {
String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
String SCRIPT_NAME = "slider"
- static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
- static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
- static final String BIN_SLIDER = "bin/slider"
- static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz"
+ String KEY_TEST_CONF_XML = "slider.test.conf.xml"
+ String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+ String BIN_SLIDER = "bin/slider"
+ String AGENT_INI = "agent.ini"
+ String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
+
+ String AGENT_TAR_FILENAME = "slider-agent.tar.gz"
+ String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index 6333fad..c3d6bd6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -20,10 +20,10 @@ package org.apache.slider.funtest.lifecycle
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.security.AccessControlException
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentUploads
import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -53,37 +53,29 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
- LOCAL_SLIDER_AGENT_TARGZ = new File(
- SLIDER_BIN_DIRECTORY,
- AGENT_SLIDER_GZ).canonicalFile
LOCAL_AGENT_CONF = new File(AGENT_CONF).canonicalFile
}
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ public static void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
- try {
- // Upload the agent tarball
- assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
- agentTarballPath = new Path(clusterFS.homeDirectory, "slider-agent.tar.gz")
- Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
- clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
-
- // Upload the agent.ini
- assume(LOCAL_AGENT_CONF.exists(), "Agent config not found at $LOCAL_AGENT_CONF")
- agtIniPath = new Path(clusterFS.homeDirectory, "agent.ini")
- Path localAgtIni = new Path(LOCAL_AGENT_CONF.toURI());
- clusterFS.copyFromLocalFile(false, true, localAgtIni, agtIniPath)
- } catch (AccessControlException ace) {
- log.info "No write access to test user home directory. " +
- "Ensure home directory exists and has correct permissions." + ace.getMessage()
- fail("Ensure home directory exists and has correct permissions for test user.")
- }
}
+
+ @Before
+ public void uploadAgentTarball() {
+ def agentUploads = new AgentUploads(SLIDER_CONFIG)
+ (agentTarballPath, agtIniPath) =
+ agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false)
+ }
@Before
@@ -105,11 +97,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath)
}
- public static void assumeAgentTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(AGENTTESTS_ENABLED, "Agent tests disabled")
- }
-
public static void logShell(SliderShell shell) {
for (String str in shell.out) {
log.info str
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
index 3db6c5b..1907a2c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
@@ -42,7 +42,7 @@ public class TestClusterBuildDestroy extends AgentCommandTestBase
@BeforeClass
public static void prepareCluster() {
- assumeFunctionalTestsEnabled();
+
setupCluster(CLUSTER)
}
[2/6] git commit: File upload work ... security model wrong
Posted by st...@apache.org.
File upload work ... security model wrong
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8327e52a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8327e52a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8327e52a
Branch: refs/heads/develop
Commit: 8327e52ae2dec00e0a954da1804413b1ef76f93c
Parents: 81578df
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 18:42:25 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 18:42:25 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 41 +++++++++++
.../funtest/framework/FileUploader.groovy | 76 ++++++++++++++++++++
.../slider/funtest/framework/SudoClosure.groovy | 54 ++++++++++++++
3 files changed, 171 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
new file mode 100644
index 0000000..b1c29c0
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+
+@Slf4j
+class AgentUploads {
+ final Configuration conf
+
+
+ AgentUploads(Configuration conf) {
+ this.conf = conf
+ }
+
+ def execUploadSequence() {
+ UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
new file mode 100644
index 0000000..dd42473
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+
+@Slf4j
+class FileUploader {
+ final Configuration conf
+ final UserGroupInformation user
+
+ FileUploader(Configuration conf, UserGroupInformation user) {
+ this.conf = conf
+ this.user = user
+ }
+
+ /**
+ * Copy if the file is considered out of date
+ * @param src
+ * @param dest
+ * @param force
+ * @return
+ */
+ public boolean copyIfOutOfDate(File src, Path dest, boolean force) {
+ def srcLen = src.length()
+ def fs = getFileSystem(user, dest.toUri())
+ boolean toCopy = force
+ if (!toCopy) {
+ try {
+ def status = fs.getFileStatus(dest)
+ toCopy = status.len != srcLen
+ } catch (FileNotFoundException fnfe) {
+ toCopy = true;
+ }
+ }
+ if (toCopy) {
+ log.info("Copying $src to $dest")
+ fs.mkdirs(dest, FsPermission.dirDefault)
+ return FileUtil.copy(src, fs, dest, false, conf)
+ } else {
+ log.debug("Skipping copy as the destination $dest considered up to date")
+ return false;
+ }
+
+ }
+
+
+ public static def getFileSystem(
+ UserGroupInformation user, final URI uri) {
+ SudoClosure.sudo(user) {
+ org.apache.hadoop.fs.FileSystem.get(uri, conf);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
new file mode 100644
index 0000000..363e1b3
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import org.apache.hadoop.security.UserGroupInformation
+
+import java.security.PrivilegedExceptionAction
+
+/**
+ * Bridge from groovy closures to doAs
+ * @param < T >
+ */
+class SudoClosure<T> implements PrivilegedExceptionAction<T> {
+
+ final Closure<T> closure;
+
+ SudoClosure(Closure<T> closure) {
+ this.closure = closure
+ }
+
+ @Override
+ T run() throws Exception {
+ return closure()
+ }
+
+ /**
+ *
+ * @param user
+ * @param closure
+ * @return
+ */
+ public static <T2> T2 sudo(UserGroupInformation user,
+ Closure<T2> closure) {
+
+ user.doAs(new SudoClosure<T2>(closure))
+
+ }
+}
[4/6] git commit: SLIDER-160: move test for file existence into
upload method; now covers all source files
Posted by st...@apache.org.
SLIDER-160: move test for file existence into upload method; now covers all source files
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eba97da0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eba97da0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eba97da0
Branch: refs/heads/develop
Commit: eba97da01765ecf924d0a4f672e8ffd2474b7464
Parents: 5464584
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 17:42:36 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 17:42:36 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/funtest/framework/AgentUploads.groovy | 2 --
.../org/apache/slider/funtest/framework/FileUploader.groovy | 3 +++
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba97da0/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index 5f10c0e..3570183 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -48,8 +48,6 @@ class AgentUploads implements FuntestProperties {
*/
def uploadAgentFiles(File tarballDir, boolean force) {
def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR)
- assert localAgentTar.exists()
-
def agentTarballPath = new Path(
homeDir,
AGENT_TAR_FILENAME)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba97da0/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 262693c..4f61730 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -44,6 +44,9 @@ class FileUploader {
* @return
*/
public boolean copyIfOutOfDate(File src, Path destPath, boolean force) {
+ if (!src.exists()) {
+ throw new FileNotFoundException("Source file $src not found")
+ }
def srcLen = src.length()
def fs = getFileSystem(destPath)
boolean toCopy = force