You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/06/30 17:36:25 UTC
[1/3] git commit: SLIDER-162: home dir created by hdfs if user lacks
permissions (and hdfs can be impersonated)
Repository: incubator-slider
Updated Branches:
refs/heads/develop 0ad4fb4f7 -> 3fb598fba
SLIDER-162: home dir created by hdfs if user lacks permissions (and hdfs can be impersonated)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e3c5d9d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e3c5d9d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e3c5d9d8
Branch: refs/heads/develop
Commit: e3c5d9d8e7198e3a30027a412b9348926900c14a
Parents: 9619c33
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 30 16:14:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 30 16:14:15 2014 +0100
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 13 ++---
.../funtest/framework/FileUploader.groovy | 51 +++++++++++++++++---
.../lifecycle/AgentCommandTestBase.groovy | 10 +++-
3 files changed, 57 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index 3570183..2cec5c2 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -27,11 +27,9 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
class AgentUploads implements FuntestProperties {
final Configuration conf
- private final FileUploader uploader
- private final HadoopFS clusterFS
- private final Path homeDir
-
-
+ public final FileUploader uploader
+ public final HadoopFS clusterFS
+ public final Path homeDir
AgentUploads(Configuration conf) {
this.conf = conf
@@ -52,6 +50,8 @@ class AgentUploads implements FuntestProperties {
homeDir,
AGENT_TAR_FILENAME)
+ //create the home dir or fail
+ uploader.mkHomeDir()
// Upload the agent tarball
uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force)
@@ -59,10 +59,7 @@ class AgentUploads implements FuntestProperties {
// Upload the agent.ini
def agentIniPath = new Path(homeDir, AGENT_INI)
uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force)
-
return [agentTarballPath, agentIniPath]
-
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 4f61730..2dc85be 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -20,12 +20,14 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.security.AccessControlException
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.fs.FileSystem as HadoopFS
+
+@SuppressWarnings("GroovyOctalInteger")
@Slf4j
class FileUploader {
final Configuration conf
@@ -60,18 +62,20 @@ class FileUploader {
}
if (toCopy) {
log.info("Copying $src to $destPath")
+ def dir = destPath.getParent()
try {
fs.delete(destPath, true)
- fs.mkdirs(destPath.getParent(), FsPermission.dirDefault)
+ fs.mkdirs(dir, FsPermission.dirDefault)
return FileUtil.copy(src, fs, destPath, false, conf)
} catch (AccessControlException ace) {
- log.error("No write access to test user home directory. " +
+ log.error("No write access to destination directory $dir" +
"Ensure home directory exists and has correct permissions." +
ace, ace)
throw ace
}
} else {
- log.debug("Skipping copy as the destination $destPath considered up to date")
+ log.debug(
+ "Skipping copy as the destination $destPath considered up to date")
return false;
}
}
@@ -79,7 +83,7 @@ class FileUploader {
public HadoopFS getFileSystem(Path dest) {
getFileSystem(user, dest)
}
-
+
public HadoopFS getFileSystem() {
getFileSystem(user, HadoopFS.getDefaultUri(conf))
}
@@ -88,13 +92,46 @@ class FileUploader {
public def getFileSystem(
UserGroupInformation user, final Path path) {
return getFileSystem(user, path.toUri())
-
+
}
+
public def getFileSystem(
UserGroupInformation user, final URI uri) {
-
+
SudoClosure.sudo(user) {
HadoopFS.get(uri, conf);
}
}
+
+ public def getFileSystemAsUserHdfs() {
+ def hdfs = UserGroupInformation.createRemoteUser("hdfs")
+ getFileSystem(hdfs, HadoopFS.getDefaultUri(conf))
+ }
+
+ /**
+ * Create the home dir. If it can't be created as the user,
+ * try to become the user 'hdfs' and try there, setting the
+ * user and group after.
+ * @return the home dir
+ */
+ public def mkHomeDir() {
+ def fs = fileSystem
+ def home = fs.homeDirectory
+ if (!fs.exists(home)) {
+ try {
+ fs.mkdirs(home)
+ } catch (AccessControlException ace) {
+ log.info("Failed to mkdir $home as $user -impersonating 'hdfs")
+ if (UserGroupInformation.securityEnabled) {
+ // in a secure cluster, we cannot impersonate HDFS, so rethrow
+ throw ace;
+ }
+ //now create as hdfs
+ def FsAsUserHDFS = fileSystemAsUserHdfs
+ FsAsUserHDFS.mkdirs(home, new FsPermission((short) 00755))
+ FsAsUserHDFS.setOwner(home, user.userName, user.primaryGroupName)
+ }
+ }
+ return home
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index c3d6bd6..33c0b81 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -80,6 +80,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Before
public void setupApplicationPackage() {
+ AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG)
+ agentUploads.uploader.mkHomeDir()
+
appPkgPath = new Path(clusterFS.homeDirectory, "cmd_log_app_pkg.zip")
if (!clusterFS.exists(appPkgPath)) {
clusterFS.delete(appPkgPath, false)
@@ -87,14 +90,17 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
def pkgPath = folder.newFolder("testpkg")
File zipFileName = new File(pkgPath, "cmd_log_app_pkg.zip").canonicalFile
- assume(new File(APP_PKG_DIR).exists(), "App pkg dir not found at $APP_PKG_DIR")
+
+ def localZipDirectory = new File(APP_PKG_DIR)
+ assume(localZipDirectory.exists(), "App pkg dir not found at $APP_PKG_DIR")
zipDir(zipFileName.canonicalPath, APP_PKG_DIR)
// Verify and upload the app pkg
assume(zipFileName.exists(), "App pkg not found at $zipFileName")
Path localAppPkg = new Path(zipFileName.toURI());
- clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath)
+ agentUploads.uploader.copyIfOutOfDate(zipFileName, appPkgPath, false)
+
}
public static void logShell(SliderShell shell) {
[3/3] git commit: Merge branch
'feature/SLIDER-162_test_runner_to_bootstrap_clean_HDFS_cluster_for_test_runs'
into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-162_test_runner_to_bootstrap_clean_HDFS_cluster_for_test_runs' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3fb598fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3fb598fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3fb598fb
Branch: refs/heads/develop
Commit: 3fb598fba45b08b1d1d13124bf3ce5a0b2c77bb5
Parents: 0ad4fb4 1bdf134
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 30 16:36:17 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 30 16:36:17 2014 +0100
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 13 ++---
.../funtest/framework/FileUploader.groovy | 51 +++++++++++++++++---
.../lifecycle/AgentCommandTestBase.groovy | 10 +++-
3 files changed, 57 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[2/3] git commit: Merge branch 'develop' into
feature/SLIDER-162_test_runner_to_bootstrap_clean_HDFS_cluster_for_test_runs
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-162_test_runner_to_bootstrap_clean_HDFS_cluster_for_test_runs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1bdf134a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1bdf134a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1bdf134a
Branch: refs/heads/develop
Commit: 1bdf134af5dd9fdb8186c848df8806324296f9cc
Parents: e3c5d9d 0ad4fb4
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 30 16:18:14 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 30 16:18:14 2014 +0100
----------------------------------------------------------------------
LICENSE | 423 +++++++++++++++++++
LICENSE.txt | 423 -------------------
NOTICE | 5 +
NOTICE.txt | 5 -
app-packages/accumulo/metainfo.xml | 238 +++++------
app-packages/hbase/metainfo.xml | 168 ++++----
app-packages/storm-v0_91/metainfo.xml | 229 +++++-----
slider-assembly/src/main/scripts/slider.py | 107 +++--
.../providers/agent/AgentClientProvider.java | 29 +-
.../slider/providers/agent/AgentKeys.java | 1 -
.../providers/agent/AgentProviderService.java | 53 +--
.../agent/application/metadata/Application.java | 121 ++++++
.../agent/application/metadata/Metainfo.java | 17 +-
.../application/metadata/MetainfoParser.java | 10 +-
.../agent/application/metadata/Service.java | 121 ------
.../appmaster/state/ContainerPriority.java | 8 +-
.../appmaster/state/OutstandingRequest.java | 7 +-
.../server/appmaster/state/RoleHistory.java | 8 +
.../app_packages/test_command_log/metainfo.xml | 62 ++-
.../agent/AgentMiniClusterTestBase.groovy | 41 +-
.../slider/providers/agent/AgentTestBase.groovy | 44 +-
.../agent/TestAgentAMManagementWS.groovy | 3 +-
.../slider/providers/agent/TestAgentEcho.groovy | 6 +-
.../providers/agent/TestBuildBasicAgent.groovy | 78 ++--
.../curator/TestRegistryRestResources.groovy | 3 +-
.../model/appstate/TestMockRMOperations.groovy | 2 +-
.../publisher/TestPublisherRestResources.groovy | 3 +-
.../slider/common/tools/TestSliderUtils.java | 23 +-
.../agent/TestAgentClientProvider.java | 40 +-
.../agent/TestAgentProviderService.java | 61 ++-
.../metadata/MetainfoParserTest.java | 15 +-
.../org/apache/slider/tools/TestUtility.java | 71 ++++
slider-core/src/test/python/appdef_1.zip | Bin 972 -> 0 bytes
slider-core/src/test/python/metainfo.xml | 68 ++-
.../org/apache/slider/common/tools/test.zip | Bin 1273 -> 0 bytes
.../slider/common/tools/test/metainfo.txt | 16 +
.../slider/common/tools/test/metainfo.xml | 95 +++++
.../slider/common/tools/test/someOtherFile.txt | 16 +
.../slider/common/tools/test/someOtherFile.xml | 16 +
.../agent/application/metadata/metainfo.xml | 136 +++---
.../slider/providers/hbase/HBaseRoles.java | 3 +-
41 files changed, 1543 insertions(+), 1232 deletions(-)
----------------------------------------------------------------------