You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2014/06/19 01:15:08 UTC
svn commit: r1603664 - in
/hadoop/common/branches/fs-encryption/hadoop-common-project: hadoop-auth/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/
hadoop-common/ hadoop-common/src/main/bin/ hadoop-common/src/main/conf/
hado...
Author: wang
Date: Wed Jun 18 23:15:04 2014
New Revision: 1603664
URL: http://svn.apache.org/r1603664
Log:
Merge trunk r1603663 to branch.
Added:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
- copied unchanged from r1603663, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/
- copied from r1603663, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
- copied unchanged from r1603663, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/
- copied from r1603663, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/pom.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/bin/hadoop
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/pom.xml?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/pom.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/pom.xml Wed Jun 18 23:15:04 2014
@@ -100,8 +100,25 @@
<dependency>
<groupId>org.apache.directory.server</groupId>
<artifactId>apacheds-kerberos-codec</artifactId>
- <version>2.0.0-M15</version>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-asn1-ber</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-i18n</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-ldap-model</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java Wed Jun 18 23:15:04 2014
@@ -193,7 +193,7 @@ public class KerberosAuthenticationHandl
for (String spnegoPrincipal : spnegoPrincipals) {
LOG.info("Login using keytab {}, for principal {}",
- keytab, principal);
+ keytab, spnegoPrincipal);
final KerberosConfiguration kerberosConfiguration =
new KerberosConfiguration(keytab, spnegoPrincipal);
final LoginContext loginContext =
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jun 18 23:15:04 2014
@@ -149,6 +149,9 @@ Trunk (Unreleased)
HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9)
+ HADOOP-10607. Create API to separate credential/password storage from
+ applications. (Larry McCay via omalley)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -427,6 +430,15 @@ Release 2.5.0 - UNRELEASED
HADOOP-10688. Expose thread-level FileSystem StatisticsData (Sandy Ryza)
+ HADOOP-10657. Have RetryInvocationHandler log failover attempt at INFO
+ level. (Ming Ma via jing9)
+
+ HADOOP-10666. Remove Copyright /d/d/d/d Apache Software Foundation from
+ the source files license header. (Henry Saputra via wang)
+
+ HADOOP-10557. FsShell -cp -pa option for preserving extended ACLs.
+ (Akira Ajisaka via cnauroth)
+
OPTIMIZATIONS
BUG FIXES
@@ -550,6 +562,17 @@ Release 2.5.0 - UNRELEASED
HADOOP-10678. SecurityUtil has unnecessary synchronization on collection
used for only tests. (Benoy Antony via cnauroth)
+ HADOOP-10683. Users authenticated with KERBEROS are recorded as being
+ authenticated with SIMPLE. (Benoy Antony via cnauroth)
+
+ HADOOP-10702. KerberosAuthenticationHandler does not log the principal names
+ correctly. (Benoy Antony via cnauroth)
+
+ HADOOP-10699. Fix build native library on mac osx (Binglin Chang via
+ jlowe)
+
+ HADOOP-10660. GraphiteSink should implement Closeable (Chen He and Ted Yu via raviprak)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
@@ -571,6 +594,10 @@ Release 2.5.0 - UNRELEASED
HADOOP-10561. Copy command with preserve option should handle Xattrs.
(Yi Liu via cnauroth)
+ HADOOP-10590. ServiceAuthorizationManager is not threadsafe. (Benoy Antony via vinayakumarb)
+
+ HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1602934-1603663
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/bin/hadoop Wed Jun 18 23:15:04 2014
@@ -116,6 +116,8 @@ case $COMMAND in
elif [ "$COMMAND" = "archive" ] ; then
CLASS=org.apache.hadoop.tools.HadoopArchives
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+ elif [ "$COMMAND" = "credential" ] ; then
+ CLASS=org.apache.hadoop.security.alias.CredentialShell
elif [[ "$COMMAND" = -* ]] ; then
# class and package names cannot begin with a -
echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh Wed Jun 18 23:15:04 2014
@@ -1,5 +1,3 @@
-# Copyright 2011 The Apache Software Foundation
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml Wed Jun 18 23:15:04 2014
@@ -1,8 +1,6 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
-
- Copyright 2011 The Apache Software Foundation
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties Wed Jun 18 23:15:04 2014
@@ -1,5 +1,3 @@
-# Copyright 2011 The Apache Software Foundation
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
Propchange: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1602934-1603663
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Jun 18 23:15:04 2014
@@ -78,6 +78,9 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
@@ -1768,6 +1771,79 @@ public class Configuration implements It
}
/**
+ * Get the value for a known password configuration element.
+ * In order to enable the elimination of clear text passwords in config,
+ * this method attempts to resolve the property name as an alias through
+ * the CredentialProvider API and conditionally fallsback to config.
+ * @param name property name
+ * @return password
+ */
+ public char[] getPassword(String name) throws IOException {
+ char[] pass = null;
+
+ pass = getPasswordFromCredenitalProviders(name);
+
+ if (pass == null) {
+ pass = getPasswordFromConfig(name);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Try and resolve the provided element name as a credential provider
+ * alias.
+ * @param name alias of the provisioned credential
+ * @return password or null if not found
+ * @throws IOException
+ */
+ protected char[] getPasswordFromCredenitalProviders(String name)
+ throws IOException {
+ char[] pass = null;
+ try {
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(this);
+
+ if (providers != null) {
+ for (CredentialProvider provider : providers) {
+ try {
+ CredentialEntry entry = provider.getCredentialEntry(name);
+ if (entry != null) {
+ pass = entry.getCredential();
+ break;
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Can't get key " + name + " from key provider" +
+ "of type: " + provider.getClass().getName() + ".", ioe);
+ }
+ }
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Configuration problem with provider path.", ioe);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Fallback to clear text passwords in configuration.
+ * @param name
+ * @return clear text password or null
+ */
+ protected char[] getPasswordFromConfig(String name) {
+ char[] pass = null;
+ if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, true)) {
+ String passStr = get(name);
+ if (passStr != null) {
+ pass = passStr.toCharArray();
+ }
+ }
+ return pass;
+ }
+
+ /**
* Get the socket address for <code>name</code> property as a
* <code>InetSocketAddress</code>.
* @param name property name.
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java Wed Jun 18 23:15:04 2014
@@ -26,6 +26,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.InputStream;
@@ -101,7 +103,7 @@ public class JavaKeyStoreProvider extend
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
this.uri = uri;
- path = unnestUri(uri);
+ path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
// Get the password file from the conf, if not present from the user's
// environment var
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java Wed Jun 18 23:15:04 2014
@@ -489,33 +489,6 @@ public abstract class KeyProvider {
}
/**
- * Convert a nested URI to decode the underlying path. The translation takes
- * the authority and parses it into the underlying scheme and authority.
- * For example, "myscheme://hdfs@nn/my/path" is converted to
- * "hdfs://nn/my/path".
- * @param nestedUri the URI from the nested URI
- * @return the unnested path
- */
- public static Path unnestUri(URI nestedUri) {
- String[] parts = nestedUri.getAuthority().split("@", 2);
- StringBuilder result = new StringBuilder(parts[0]);
- result.append("://");
- if (parts.length == 2) {
- result.append(parts[1]);
- }
- result.append(nestedUri.getPath());
- if (nestedUri.getQuery() != null) {
- result.append("?");
- result.append(nestedUri.getQuery());
- }
- if (nestedUri.getFragment() != null) {
- result.append("#");
- result.append(nestedUri.getFragment());
- }
- return new Path(result.toString());
- }
-
- /**
* Find the provider with the given key.
* @param providerList the list of providers
* @param keyName the key name we are looking for
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Wed Jun 18 23:15:04 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
@@ -147,7 +148,7 @@ public class KMSClientProvider extends K
}
public KMSClientProvider(URI uri, Configuration conf) throws IOException {
- Path path = unnestUri(uri);
+ Path path = ProviderUtils.unnestUri(uri);
URL url = path.toUri().toURL();
kmsUrl = createServiceURL(url);
if ("https".equalsIgnoreCase(url.getProtocol())) {
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Jun 18 23:15:04 2014
@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
@@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathIsDirect
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.PathOperationException;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
/**
@@ -88,7 +92,7 @@ abstract class CommandWithDestination ex
}
protected static enum FileAttribute {
- TIMESTAMPS, OWNERSHIP, PERMISSION, XATTR;
+ TIMESTAMPS, OWNERSHIP, PERMISSION, ACL, XATTR;
public static FileAttribute getAttribute(char symbol) {
for (FileAttribute attribute : values()) {
@@ -306,11 +310,22 @@ abstract class CommandWithDestination ex
src.stat.getOwner(),
src.stat.getGroup());
}
- if (shouldPreserve(FileAttribute.PERMISSION)) {
+ if (shouldPreserve(FileAttribute.PERMISSION) ||
+ shouldPreserve(FileAttribute.ACL)) {
target.fs.setPermission(
target.path,
src.stat.getPermission());
}
+ if (shouldPreserve(FileAttribute.ACL)) {
+ FsPermission perm = src.stat.getPermission();
+ if (perm.getAclBit()) {
+ List<AclEntry> srcEntries =
+ src.fs.getAclStatus(src.path).getEntries();
+ List<AclEntry> srcFullEntries =
+ AclUtil.getAclFromPermAndEntries(perm, srcEntries);
+ target.fs.setAcl(target.path, srcFullEntries);
+ }
+ }
if (shouldPreserve(FileAttribute.XATTR)) {
Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
if (srcXAttrs != null) {
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java Wed Jun 18 23:15:04 2014
@@ -133,16 +133,18 @@ class CopyCommands {
static class Cp extends CommandWithDestination {
public static final String NAME = "cp";
- public static final String USAGE = "[-f] [-p | -p[topx]] <src> ... <dst>";
+ public static final String USAGE = "[-f] [-p | -p[topax]] <src> ... <dst>";
public static final String DESCRIPTION =
"Copy files that match the file pattern <src> to a " +
"destination. When copying multiple files, the destination " +
"must be a directory. Passing -p preserves status " +
- "[topx] (timestamps, ownership, permission, XAttr). " +
+ "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
"If -p is specified with no <arg>, then preserves " +
- "timestamps, ownership, permission. Passing -f " +
- "overwrites the destination if it already exists.\n";
-
+ "timestamps, ownership, permission. If -pa is specified, " +
+ "then preserves permission also because ACL is a super-set of " +
+ "permission. Passing -f overwrites the destination if it " +
+ "already exists.\n";
+
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
popPreserveOption(args);
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java Wed Jun 18 23:15:04 2014
@@ -136,9 +136,7 @@ public class RetryInvocationHandler<T> i
msg += " after " + invocationFailoverCount + " fail over attempts";
}
msg += ". Trying to fail over " + formatSleepMessage(action.delayMillis);
- if (LOG.isDebugEnabled()) {
- LOG.debug(msg, e);
- }
+ LOG.info(msg, e);
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("Exception while invoking " + method.getName()
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Wed Jun 18 23:15:04 2014
@@ -1221,7 +1221,7 @@ public abstract class Server {
ugi.addTokenIdentifier(tokenId);
return ugi;
} else {
- return UserGroupInformation.createRemoteUser(authorizedId);
+ return UserGroupInformation.createRemoteUser(authorizedId, authMethod);
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java Wed Jun 18 23:15:04 2014
@@ -18,13 +18,18 @@
package org.apache.hadoop.metrics2.sink;
+import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.io.Closeable;
import java.net.Socket;
import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
@@ -36,12 +41,14 @@ import org.apache.hadoop.metrics2.Metric
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class GraphiteSink implements MetricsSink {
+public class GraphiteSink implements MetricsSink, Closeable {
+ private static final Log LOG = LogFactory.getLog(GraphiteSink.class);
private static final String SERVER_HOST_KEY = "server_host";
private static final String SERVER_PORT_KEY = "server_port";
private static final String METRICS_PREFIX = "metrics_prefix";
private Writer writer = null;
private String metricsPrefix = null;
+ private Socket socket = null;
public void setWriter(Writer writer) {
this.writer = writer;
@@ -60,7 +67,7 @@ public class GraphiteSink implements Met
try {
// Open an connection to Graphite server.
- Socket socket = new Socket(serverHost, serverPort);
+ socket = new Socket(serverHost, serverPort);
setWriter(new OutputStreamWriter(socket.getOutputStream()));
} catch (Exception e) {
throw new MetricsException("Error creating connection, "
@@ -99,7 +106,11 @@ public class GraphiteSink implements Met
}
try {
- writer.write(lines.toString());
+ if(writer != null){
+ writer.write(lines.toString());
+ } else {
+ throw new MetricsException("Writer in GraphiteSink is null!");
+ }
} catch (Exception e) {
throw new MetricsException("Error sending metrics", e);
}
@@ -113,4 +124,21 @@ public class GraphiteSink implements Met
throw new MetricsException("Error flushing metrics", e);
}
}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ IOUtils.closeStream(writer);
+ writer = null;
+ LOG.info("writer in GraphiteSink is closed!");
+ } catch (Throwable e){
+ throw new MetricsException("Error closing writer", e);
+ } finally {
+ if (socket != null && !socket.isClosed()) {
+ socket.close();
+ socket = null;
+ LOG.info("socket in GraphiteSink is closed!");
+ }
+ }
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java Wed Jun 18 23:15:04 2014
@@ -1157,13 +1157,25 @@ public class UserGroupInformation {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation createRemoteUser(String user) {
+ return createRemoteUser(user, AuthMethod.SIMPLE);
+ }
+
+ /**
+ * Create a user from a login name. It is intended to be used for remote
+ * users in RPC, since it won't have any credentials.
+ * @param user the full user principal name, must not be empty or null
+ * @return the UserGroupInformation for the remote user.
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public static UserGroupInformation createRemoteUser(String user, AuthMethod authMethod) {
if (user == null || user.isEmpty()) {
throw new IllegalArgumentException("Null user");
}
Subject subject = new Subject();
subject.getPrincipals().add(new User(user));
UserGroupInformation result = new UserGroupInformation(subject);
- result.setAuthenticationMethod(AuthenticationMethod.SIMPLE);
+ result.setAuthenticationMethod(authMethod);
return result;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java Wed Jun 18 23:15:04 2014
@@ -45,7 +45,7 @@ import com.google.common.annotations.Vis
public class ServiceAuthorizationManager {
private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml";
- private Map<Class<?>, AccessControlList> protocolToAcl =
+ private volatile Map<Class<?>, AccessControlList> protocolToAcl =
new IdentityHashMap<Class<?>, AccessControlList>();
/**
@@ -114,7 +114,7 @@ public class ServiceAuthorizationManager
AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol);
}
- public synchronized void refresh(Configuration conf,
+ public void refresh(Configuration conf,
PolicyProvider provider) {
// Get the system property 'hadoop.policy.file'
String policyFile =
@@ -127,7 +127,7 @@ public class ServiceAuthorizationManager
}
@Private
- public synchronized void refreshWithLoadedConfiguration(Configuration conf,
+ public void refreshWithLoadedConfiguration(Configuration conf,
PolicyProvider provider) {
final Map<Class<?>, AccessControlList> newAcls =
new IdentityHashMap<Class<?>, AccessControlList>();
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c Wed Jun 18 23:15:04 2014
@@ -73,7 +73,10 @@ Java_org_apache_hadoop_security_JniBased
// was successful or not (as long as it was called we need to call
// endnetgrent)
setnetgrentCalledFlag = 1;
-#ifndef __FreeBSD__
+#if defined(__FreeBSD__) || defined(__MACH__)
+ setnetgrent(cgroup);
+ {
+#else
if(setnetgrent(cgroup) == 1) {
#endif
current = NULL;
@@ -90,9 +93,7 @@ Java_org_apache_hadoop_security_JniBased
userListSize++;
}
}
-#ifndef __FreeBSD__
}
-#endif
//--------------------------------------------------
// build return data (java array)
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm Wed Jun 18 23:15:04 2014
@@ -306,9 +306,9 @@ Administration Commands
Runs a cluster balancing utility. An administrator can simply press Ctrl-C
to stop the rebalancing process. See
- {{{../hadoop-hdfs/HdfsUserGuide.html#Rebalancer}Rebalancer}} for more details.
+ {{{../hadoop-hdfs/HdfsUserGuide.html#Balancer}Balancer}} for more details.
- Usage: <<<hadoop balancer [-threshold <threshold>]>>>
+ Usage: <<<hadoop balancer [-threshold <threshold>] [-policy <policy>]>>>
*------------------------+-----------------------------------------------------------+
|| COMMAND_OPTION | Description
@@ -316,6 +316,11 @@ Administration Commands
| -threshold <threshold> | Percentage of disk capacity. This overwrites the
| default threshold.
*------------------------+-----------------------------------------------------------+
+| -policy <policy> | <<<datanode>>> (default): Cluster is balanced if each datanode is balanced. \
+ | <<<blockpool>>>: Cluster is balanced if each block pool in each datanode is balanced.
+*------------------------+-----------------------------------------------------------+
+
+ Note that the <<<blockpool>>> policy is more strict than the <<<datanode>>> policy.
* <<<daemonlog>>>
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Wed Jun 18 23:15:04 2014
@@ -159,7 +159,7 @@ count
cp
- Usage: <<<hdfs dfs -cp [-f] [-p | -p[topx]] URI [URI ...] <dest> >>>
+ Usage: <<<hdfs dfs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> >>>
Copy files from source to destination. This command allows multiple sources
as well in which case the destination must be a directory.
@@ -169,8 +169,10 @@ cp
* The -f option will overwrite the destination if it already exists.
* The -p option will preserve file attributes [topx] (timestamps,
- ownership, permission, XAttr). If -p is specified with no <arg>,
- then preserves timestamps, ownership, permission.
+ ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
+ then preserves timestamps, ownership, permission. If -pa is specified,
+ then preserves permission also because ACL is a super-set of
+ permission.
Example:
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java Wed Jun 18 23:15:04 2014
@@ -21,6 +21,7 @@ import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.junit.Test;
import java.io.IOException;
@@ -133,13 +134,13 @@ public class TestKeyProvider {
@Test
public void testUnnestUri() throws Exception {
assertEquals(new Path("hdfs://nn.example.com/my/path"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
assertEquals(new Path("hdfs://nn/my/path?foo=bar&baz=bat#yyy"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
assertEquals(new Path("inner://hdfs@nn1.example.com/my/path"),
- KeyProvider.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
assertEquals(new Path("user:///"),
- KeyProvider.unnestUri(new URI("outer://user/")));
+ ProviderUtils.unnestUri(new URI("outer://user/")));
}
private static class MyKeyProvider extends KeyProvider {
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java Wed Jun 18 23:15:04 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Before;
@@ -213,7 +214,7 @@ public class TestKeyProviderFactory {
file.delete();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
checkSpecificProvider(conf, ourUrl);
- Path path = KeyProvider.unnestUri(new URI(ourUrl));
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------"));
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java Wed Jun 18 23:15:04 2014
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -30,6 +31,7 @@ import java.util.Set;
import static org.mockito.Mockito.*;
import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.sink.GraphiteSink;
@@ -107,4 +109,39 @@ public class TestGraphiteMetrics {
result.equals("null.all.Context.Context=all.foo2 2 10\n" +
"null.all.Context.Context=all.foo1 1 10\n"));
}
+ @Test(expected=MetricsException.class)
+ public void testCloseAndWrite() throws IOException {
+ GraphiteSink sink = new GraphiteSink();
+ List<MetricsTag> tags = new ArrayList<MetricsTag>();
+ tags.add(new MetricsTag(MsInfo.Context, "all"));
+ tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+ Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
+ metrics.add(makeMetric("foo1", 1.25));
+ metrics.add(makeMetric("foo2", 2.25));
+ MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
+
+ OutputStreamWriter writer = mock(OutputStreamWriter.class);
+
+ sink.setWriter(writer);
+ sink.close();
+ sink.putMetrics(record);
+ }
+
+ @Test
+ public void testClose(){
+ GraphiteSink sink = new GraphiteSink();
+ Writer mockWriter = mock(Writer.class);
+ sink.setWriter(mockWriter);
+ try {
+ sink.close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+
+ try {
+ verify(mockWriter).close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Wed Jun 18 23:15:04 2014
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
@@ -31,6 +32,7 @@ import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -151,6 +153,18 @@ public class TestUserGroupInformation {
assertEquals(AuthenticationMethod.PROXY, ugi.getAuthenticationMethod());
assertEquals(AuthenticationMethod.SIMPLE, ugi.getRealAuthenticationMethod());
}
+
+ @Test (timeout = 30000)
+ public void testCreateRemoteUser() {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user1");
+ assertEquals(AuthenticationMethod.SIMPLE, ugi.getAuthenticationMethod());
+ assertTrue (ugi.toString().contains("(auth:SIMPLE)"));
+ ugi = UserGroupInformation.createRemoteUser("user1",
+ AuthMethod.KERBEROS);
+ assertEquals(AuthenticationMethod.KERBEROS, ugi.getAuthenticationMethod());
+ assertTrue (ugi.toString().contains("(auth:KERBEROS)"));
+ }
+
/** Test login method */
@Test (timeout = 30000)
public void testLogin() throws Exception {
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml Wed Jun 18 23:15:04 2014
@@ -296,7 +296,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-cp \[-f\] \[-p \| -p\[topx\]\] <src> \.\.\. <dst> :\s*</expected-output>
+ <expected-output>^-cp \[-f\] \[-p \| -p\[topax\]\] <src> \.\.\. <dst> :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -308,15 +308,19 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*\[topx\] \(timestamps, ownership, permission, XAttr\). If -p is specified with no( )*</expected-output>
+ <expected-output>^( |\t)*\[topax\] \(timestamps, ownership, permission, ACLs, XAttr\). If -p is specified( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^( |\t)*<arg>, then preserves timestamps, ownership, permission. Passing -f overwrites( )*</expected-output>
+ <expected-output>^( |\t)*with no <arg>, then preserves timestamps, ownership, permission. If -pa is( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*the destination if it already exists.( )*</expected-output>
+ <expected-output>^( |\t)*specified, then preserves permission also because ACL is a super-set of( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^\s*permission. Passing -f overwrites the destination if it already exists.( )*</expected-output>
</comparator>
</comparators>
</test>
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java Wed Jun 18 23:15:04 2014
@@ -17,12 +17,9 @@
*/
package org.apache.hadoop.nfs.nfs3;
-import java.net.InetAddress;
-
import org.apache.hadoop.nfs.nfs3.response.NFS3Response;
+import org.apache.hadoop.oncrpc.RpcInfo;
import org.apache.hadoop.oncrpc.XDR;
-import org.apache.hadoop.oncrpc.security.SecurityHandler;
-import org.jboss.netty.channel.Channel;
/**
* RPC procedures as defined in RFC 1813.
@@ -33,70 +30,65 @@ public interface Nfs3Interface {
public NFS3Response nullProcedure();
/** GETATTR: Get file attributes */
- public NFS3Response getattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response getattr(XDR xdr, RpcInfo info);
/** SETATTR: Set file attributes */
- public NFS3Response setattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response setattr(XDR xdr, RpcInfo info);
/** LOOKUP: Lookup filename */
- public NFS3Response lookup(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response lookup(XDR xdr, RpcInfo info);
/** ACCESS: Check access permission */
- public NFS3Response access(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response access(XDR xdr, RpcInfo info);
+
+ /** READLINK: Read from symbolic link */
+ public NFS3Response readlink(XDR xdr, RpcInfo info);
/** READ: Read from file */
- public NFS3Response read(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response read(XDR xdr, RpcInfo info);
/** WRITE: Write to file */
- public NFS3Response write(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client);
+ public NFS3Response write(XDR xdr, RpcInfo info);
/** CREATE: Create a file */
- public NFS3Response create(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response create(XDR xdr, RpcInfo info);
/** MKDIR: Create a directory */
- public NFS3Response mkdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response mkdir(XDR xdr, RpcInfo info);
+
+ /** SYMLINK: Create a symbolic link */
+ public NFS3Response symlink(XDR xdr, RpcInfo info);
+
+ /** MKNOD: Create a special device */
+ public NFS3Response mknod(XDR xdr, RpcInfo info);
/** REMOVE: Remove a file */
- public NFS3Response remove(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response remove(XDR xdr, RpcInfo info);
/** RMDIR: Remove a directory */
- public NFS3Response rmdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response rmdir(XDR xdr, RpcInfo info);
/** RENAME: Rename a file or directory */
- public NFS3Response rename(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response rename(XDR xdr, RpcInfo info);
- /** SYMLINK: Create a symbolic link */
- public NFS3Response symlink(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ /** LINK: create link to an object */
+ public NFS3Response link(XDR xdr, RpcInfo info);
/** READDIR: Read From directory */
- public NFS3Response readdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response readdir(XDR xdr, RpcInfo info);
+ /** READDIRPLUS: Extended read from directory */
+ public NFS3Response readdirplus(XDR xdr, RpcInfo info);
+
/** FSSTAT: Get dynamic file system information */
- public NFS3Response fsstat(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response fsstat(XDR xdr, RpcInfo info);
/** FSINFO: Get static file system information */
- public NFS3Response fsinfo(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response fsinfo(XDR xdr, RpcInfo info);
/** PATHCONF: Retrieve POSIX information */
- public NFS3Response pathconf(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response pathconf(XDR xdr, RpcInfo info);
/** COMMIT: Commit cached data on a server to stable storage */
- public NFS3Response commit(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client);
+ public NFS3Response commit(XDR xdr, RpcInfo info);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java Wed Jun 18 23:15:04 2014
@@ -48,7 +48,7 @@ public abstract class RpcProgram extends
private final int progNumber;
private final int lowProgVersion;
private final int highProgVersion;
- private final boolean allowInsecurePorts;
+ protected final boolean allowInsecurePorts;
/**
* If not null, this will be used as the socket to use to connect to the
@@ -146,31 +146,6 @@ public abstract class RpcProgram extends
RpcCall call = (RpcCall) info.header();
SocketAddress remoteAddress = info.remoteAddress();
- if (!allowInsecurePorts) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Will not allow connections from unprivileged ports. " +
- "Checking for valid client port...");
- }
- if (remoteAddress instanceof InetSocketAddress) {
- InetSocketAddress inetRemoteAddress = (InetSocketAddress) remoteAddress;
- if (inetRemoteAddress.getPort() > 1023) {
- LOG.warn("Connection attempted from '" + inetRemoteAddress + "' "
- + "which is an unprivileged port. Rejecting connection.");
- sendRejectedReply(call, remoteAddress, ctx);
- return;
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Accepting connection from '" + remoteAddress + "'");
- }
- }
- } else {
- LOG.warn("Could not determine remote port of socket address '" +
- remoteAddress + "'. Rejecting connection.");
- sendRejectedReply(call, remoteAddress, ctx);
- return;
- }
- }
-
if (LOG.isTraceEnabled()) {
LOG.trace(program + " procedure #" + call.getProcedure());
}
@@ -191,6 +166,29 @@ public abstract class RpcProgram extends
handleInternal(ctx, info);
}
+ public boolean doPortMonitoring(SocketAddress remoteAddress) {
+ if (!allowInsecurePorts) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Will not allow connections from unprivileged ports. "
+ + "Checking for valid client port...");
+ }
+
+ if (remoteAddress instanceof InetSocketAddress) {
+ InetSocketAddress inetRemoteAddress = (InetSocketAddress) remoteAddress;
+ if (inetRemoteAddress.getPort() > 1023) {
+ LOG.warn("Connection attempted from '" + inetRemoteAddress + "' "
+ + "which is an unprivileged port. Rejecting connection.");
+ return false;
+ }
+ } else {
+ LOG.warn("Could not determine remote port of socket address '"
+ + remoteAddress + "'. Rejecting connection.");
+ return false;
+ }
+ }
+ return true;
+ }
+
private void sendAcceptedReply(RpcCall call, SocketAddress remoteAddress,
AcceptState acceptState, ChannelHandlerContext ctx) {
RpcAcceptedReply reply = RpcAcceptedReply.getInstance(call.getXid(),
@@ -208,7 +206,7 @@ public abstract class RpcProgram extends
RpcUtil.sendRpcResponse(ctx, rsp);
}
- private static void sendRejectedReply(RpcCall call,
+ protected static void sendRejectedReply(RpcCall call,
SocketAddress remoteAddress, ChannelHandlerContext ctx) {
XDR out = new XDR();
RpcDeniedReply reply = new RpcDeniedReply(call.getXid(),
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java?rev=1603664&r1=1603663&r2=1603664&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java Wed Jun 18 23:15:04 2014
@@ -66,6 +66,18 @@ public class TestFrameDecoder {
@Override
protected void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
+ // This is just like what's done in RpcProgramMountd#handleInternal and
+ // RpcProgramNfs3#handleInternal.
+ RpcCall rpcCall = (RpcCall) info.header();
+ final int procedure = rpcCall.getProcedure();
+ if (procedure != 0) {
+ boolean portMonitorSuccess = doPortMonitoring(info.remoteAddress());
+ if (!portMonitorSuccess) {
+ sendRejectedReply(rpcCall, info.remoteAddress(), ctx);
+ return;
+ }
+ }
+
resultSize = info.data().readableBytes();
RpcAcceptedReply reply = RpcAcceptedReply.getAcceptInstance(1234,
new VerifierNone());
@@ -190,6 +202,20 @@ public class TestFrameDecoder {
// Verify the server rejected the request.
assertEquals(0, resultSize);
+
+ // Ensure that the NULL procedure does in fact succeed.
+ xdrOut = new XDR();
+ createPortmapXDRheader(xdrOut, 0);
+ int headerSize = xdrOut.size();
+ buffer = new byte[bufsize];
+ xdrOut.writeFixedOpaque(buffer);
+ int requestSize = xdrOut.size() - headerSize;
+
+ // Send the request to the server
+ testRequest(xdrOut, serverPort);
+
+ // Verify the server did not reject the request.
+ assertEquals(requestSize, resultSize);
}
private static int startRpcServer(boolean allowInsecurePorts) {