You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/06/25 03:11:59 UTC
[3/6] git commit: SLIDER-160: agent functional tests upload
binaries/ini file if there is a difference
SLIDER-160: agent functional tests upload binaries/ini file if there is a difference
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5464584d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5464584d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5464584d
Branch: refs/heads/develop
Commit: 5464584d784cf57348bf22100cee2581d5a887c0
Parents: 8327e52
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 17:40:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 17:40:49 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 41 +++++++++++++++---
.../funtest/framework/CommandTestBase.groovy | 9 ++--
.../funtest/framework/FileUploader.groovy | 45 ++++++++++++++------
.../funtest/framework/FuntestProperties.groovy | 12 ++++--
.../lifecycle/AgentCommandTestBase.groovy | 39 ++++++-----------
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
6 files changed, 94 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index b1c29c0..5f10c0e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -20,22 +20,51 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
-class AgentUploads {
+class AgentUploads implements FuntestProperties {
final Configuration conf
+ private final FileUploader uploader
+ private final HadoopFS clusterFS
+ private final Path homeDir
+
AgentUploads(Configuration conf) {
this.conf = conf
+ uploader = new FileUploader(conf, UserGroupInformation.currentUser)
+ clusterFS = uploader.fileSystem
+ homeDir = clusterFS.homeDirectory
}
- def execUploadSequence() {
- UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn")
+ /**
+ * Upload agent-related files
+ * @param tarballDir
+ * @param force
+ * @return
+ */
+ def uploadAgentFiles(File tarballDir, boolean force) {
+ def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR)
+ assert localAgentTar.exists()
+
+ def agentTarballPath = new Path(
+ homeDir,
+ AGENT_TAR_FILENAME)
+
+ // Upload the agent tarball
+ uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force)
+
+ File localAgentIni = new File(tarballDir, AGENT_INI_IN_SLIDER_TAR)
+ // Upload the agent.ini
+ def agentIniPath = new Path(homeDir, AGENT_INI)
+ uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force)
+
+ return [agentTarballPath, agentIniPath]
+
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d2accbb..61ae804 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -50,17 +50,16 @@ abstract class CommandTestBase extends SliderTestUtils {
LoggerFactory.getLogger(CommandTestBase.class);
public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP)
- public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP)
- public static final File SLIDER_BIN_DIRECTORY = new File(
- SLIDER_BIN_DIR).canonicalFile
+ public static final String SLIDER_TAR_DIR = sysprop(SLIDER_BIN_DIR_PROP)
+ public static final File SLIDER_TAR_DIRECTORY = new File(
+ SLIDER_TAR_DIR).canonicalFile
public static final File SLIDER_SCRIPT = new File(
- SLIDER_BIN_DIRECTORY,
+ SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
-
public static final YarnConfiguration SLIDER_CONFIG
public static final int THAW_WAIT_TIME
public static final int FREEZE_WAIT_TIME
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index dd42473..262693c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -23,8 +23,9 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.AccessControlException
import org.apache.hadoop.security.UserGroupInformation
-
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
class FileUploader {
final Configuration conf
@@ -38,39 +39,59 @@ class FileUploader {
/**
* Copy if the file is considered out of date
* @param src
- * @param dest
+ * @param destPath
* @param force
* @return
*/
- public boolean copyIfOutOfDate(File src, Path dest, boolean force) {
+ public boolean copyIfOutOfDate(File src, Path destPath, boolean force) {
def srcLen = src.length()
- def fs = getFileSystem(user, dest.toUri())
+ def fs = getFileSystem(destPath)
boolean toCopy = force
if (!toCopy) {
try {
- def status = fs.getFileStatus(dest)
+ def status = fs.getFileStatus(destPath)
toCopy = status.len != srcLen
} catch (FileNotFoundException fnfe) {
toCopy = true;
}
}
if (toCopy) {
- log.info("Copying $src to $dest")
- fs.mkdirs(dest, FsPermission.dirDefault)
- return FileUtil.copy(src, fs, dest, false, conf)
+ log.info("Copying $src to $destPath")
+ try {
+ fs.delete(destPath, true)
+ fs.mkdirs(destPath.getParent(), FsPermission.dirDefault)
+ return FileUtil.copy(src, fs, destPath, false, conf)
+ } catch (AccessControlException ace) {
+ log.error("No write access to test user home directory. " +
+ "Ensure home directory exists and has correct permissions." +
+ ace, ace)
+ throw ace
+ }
} else {
- log.debug("Skipping copy as the destination $dest considered up to date")
+ log.debug("Skipping copy as the destination $destPath considered up to date")
return false;
}
+ }
+ public HadoopFS getFileSystem(Path dest) {
+ getFileSystem(user, dest)
+ }
+
+ public HadoopFS getFileSystem() {
+ getFileSystem(user, HadoopFS.getDefaultUri(conf))
}
- public static def getFileSystem(
+ public def getFileSystem(
+ UserGroupInformation user, final Path path) {
+ return getFileSystem(user, path.toUri())
+
+ }
+ public def getFileSystem(
UserGroupInformation user, final URI uri) {
+
SudoClosure.sudo(user) {
- org.apache.hadoop.fs.FileSystem.get(uri, conf);
+ HadoopFS.get(uri, conf);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 8cbc098..9b63c22 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,8 +57,12 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting {
String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
String SCRIPT_NAME = "slider"
- static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
- static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
- static final String BIN_SLIDER = "bin/slider"
- static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz"
+ String KEY_TEST_CONF_XML = "slider.test.conf.xml"
+ String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+ String BIN_SLIDER = "bin/slider"
+ String AGENT_INI = "agent.ini"
+ String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
+
+ String AGENT_TAR_FILENAME = "slider-agent.tar.gz"
+ String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index 6333fad..c3d6bd6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -20,10 +20,10 @@ package org.apache.slider.funtest.lifecycle
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.security.AccessControlException
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentUploads
import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -53,37 +53,29 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
- LOCAL_SLIDER_AGENT_TARGZ = new File(
- SLIDER_BIN_DIRECTORY,
- AGENT_SLIDER_GZ).canonicalFile
LOCAL_AGENT_CONF = new File(AGENT_CONF).canonicalFile
}
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ public static void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
- try {
- // Upload the agent tarball
- assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
- agentTarballPath = new Path(clusterFS.homeDirectory, "slider-agent.tar.gz")
- Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
- clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
-
- // Upload the agent.ini
- assume(LOCAL_AGENT_CONF.exists(), "Agent config not found at $LOCAL_AGENT_CONF")
- agtIniPath = new Path(clusterFS.homeDirectory, "agent.ini")
- Path localAgtIni = new Path(LOCAL_AGENT_CONF.toURI());
- clusterFS.copyFromLocalFile(false, true, localAgtIni, agtIniPath)
- } catch (AccessControlException ace) {
- log.info "No write access to test user home directory. " +
- "Ensure home directory exists and has correct permissions." + ace.getMessage()
- fail("Ensure home directory exists and has correct permissions for test user.")
- }
}
+
+ @Before
+ public void uploadAgentTarball() {
+ def agentUploads = new AgentUploads(SLIDER_CONFIG)
+ (agentTarballPath, agtIniPath) =
+ agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false)
+ }
@Before
@@ -105,11 +97,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath)
}
- public static void assumeAgentTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(AGENTTESTS_ENABLED, "Agent tests disabled")
- }
-
public static void logShell(SliderShell shell) {
for (String str in shell.out) {
log.info str
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
index 3db6c5b..1907a2c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
@@ -42,7 +42,7 @@ public class TestClusterBuildDestroy extends AgentCommandTestBase
@BeforeClass
public static void prepareCluster() {
- assumeFunctionalTestsEnabled();
+
setupCluster(CLUSTER)
}