You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2014/08/20 03:34:41 UTC
svn commit: r1619019 [3/6] - in
/hadoop/common/branches/YARN-1051/hadoop-common-project: hadoop-auth/
hadoop-auth/dev-support/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/main/java/org/apache/hadoop/secur...
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java Wed Aug 20 01:34:29 2014
@@ -38,9 +38,9 @@ import org.apache.hadoop.util.ToolRunner
*/
public class KeyShell extends Configured implements Tool {
final static private String USAGE_PREFIX = "Usage: hadoop key " +
- "[generic options]\n";
+ "[generic options]\n";
final static private String COMMANDS =
- " [--help]\n" +
+ " [-help]\n" +
" [" + CreateCommand.USAGE + "]\n" +
" [" + RollCommand.USAGE + "]\n" +
" [" + DeleteCommand.USAGE + "]\n" +
@@ -90,11 +90,11 @@ public class KeyShell extends Configured
/**
* Parse the command line arguments and initialize the data
* <pre>
- * % hadoop key create keyName [--size size] [--cipher algorithm]
- * [--provider providerPath]
- * % hadoop key roll keyName [--provider providerPath]
+ * % hadoop key create keyName [-size size] [-cipher algorithm]
+ * [-provider providerPath]
+ * % hadoop key roll keyName [-provider providerPath]
* % hadoop key list [-provider providerPath]
- * % hadoop key delete keyName [--provider providerPath] [-i]
+ * % hadoop key delete keyName [-provider providerPath] [-i]
* </pre>
* @param args Command line arguments.
* @return 0 on success, 1 on failure.
@@ -107,47 +107,47 @@ public class KeyShell extends Configured
for (int i = 0; i < args.length; i++) { // parse command line
boolean moreTokens = (i < args.length - 1);
if (args[i].equals("create")) {
- String keyName = "--help";
+ String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new CreateCommand(keyName, options);
- if ("--help".equals(keyName)) {
+ if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if (args[i].equals("delete")) {
- String keyName = "--help";
+ String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new DeleteCommand(keyName);
- if ("--help".equals(keyName)) {
+ if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if (args[i].equals("roll")) {
- String keyName = "--help";
+ String keyName = "-help";
if (moreTokens) {
keyName = args[++i];
}
command = new RollCommand(keyName);
- if ("--help".equals(keyName)) {
+ if ("-help".equals(keyName)) {
printKeyShellUsage();
return 1;
}
} else if ("list".equals(args[i])) {
command = new ListCommand();
- } else if ("--size".equals(args[i]) && moreTokens) {
+ } else if ("-size".equals(args[i]) && moreTokens) {
options.setBitLength(Integer.parseInt(args[++i]));
- } else if ("--cipher".equals(args[i]) && moreTokens) {
+ } else if ("-cipher".equals(args[i]) && moreTokens) {
options.setCipher(args[++i]);
- } else if ("--description".equals(args[i]) && moreTokens) {
+ } else if ("-description".equals(args[i]) && moreTokens) {
options.setDescription(args[++i]);
- } else if ("--attr".equals(args[i]) && moreTokens) {
+ } else if ("-attr".equals(args[i]) && moreTokens) {
final String attrval[] = args[++i].split("=", 2);
final String attr = attrval[0].trim();
final String val = attrval[1].trim();
@@ -164,14 +164,14 @@ public class KeyShell extends Configured
return 1;
}
attributes.put(attr, val);
- } else if ("--provider".equals(args[i]) && moreTokens) {
+ } else if ("-provider".equals(args[i]) && moreTokens) {
userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
- } else if ("--metadata".equals(args[i])) {
+ } else if ("-metadata".equals(args[i])) {
getConf().setBoolean(LIST_METADATA, true);
- } else if ("-i".equals(args[i]) || ("--interactive".equals(args[i]))) {
+ } else if ("-i".equals(args[i]) || ("-interactive".equals(args[i]))) {
interactive = true;
- } else if ("--help".equals(args[i])) {
+ } else if ("-help".equals(args[i])) {
printKeyShellUsage();
return 1;
} else {
@@ -258,11 +258,11 @@ public class KeyShell extends Configured
private class ListCommand extends Command {
public static final String USAGE =
- "list [--provider <provider>] [--metadata] [--help]";
+ "list [-provider <provider>] [-metadata] [-help]";
public static final String DESC =
"The list subcommand displays the keynames contained within\n" +
"a particular provider as configured in core-site.xml or\n" +
- "specified with the --provider argument. --metadata displays\n" +
+ "specified with the -provider argument. -metadata displays\n" +
"the metadata.";
private boolean metadata = false;
@@ -272,9 +272,9 @@ public class KeyShell extends Configured
provider = getKeyProvider();
if (provider == null) {
out.println("There are no non-transient KeyProviders configured.\n"
- + "Use the --provider option to specify a provider. If you\n"
+ + "Use the -provider option to specify a provider. If you\n"
+ "want to list a transient provider then you must use the\n"
- + "--provider argument.");
+ + "-provider argument.");
rc = false;
}
metadata = getConf().getBoolean(LIST_METADATA, false);
@@ -310,10 +310,10 @@ public class KeyShell extends Configured
}
private class RollCommand extends Command {
- public static final String USAGE = "roll <keyname> [--provider <provider>] [--help]";
+ public static final String USAGE = "roll <keyname> [-provider <provider>] [-help]";
public static final String DESC =
"The roll subcommand creates a new version for the specified key\n" +
- "within the provider indicated using the --provider argument\n";
+ "within the provider indicated using the -provider argument\n";
String keyName = null;
@@ -326,13 +326,13 @@ public class KeyShell extends Configured
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. The key\n" +
- "has not been rolled. Use the --provider option to specify\n" +
+ "has not been rolled. Use the -provider option to specify\n" +
"a provider.");
rc = false;
}
if (keyName == null) {
out.println("Please provide a <keyname>.\n" +
- "See the usage description by using --help.");
+ "See the usage description by using -help.");
rc = false;
}
return rc;
@@ -367,11 +367,11 @@ public class KeyShell extends Configured
}
private class DeleteCommand extends Command {
- public static final String USAGE = "delete <keyname> [--provider <provider>] [--help]";
+ public static final String USAGE = "delete <keyname> [-provider <provider>] [-help]";
public static final String DESC =
"The delete subcommand deletes all versions of the key\n" +
"specified by the <keyname> argument from within the\n" +
- "provider specified --provider.";
+ "provider specified -provider.";
String keyName = null;
boolean cont = true;
@@ -385,12 +385,12 @@ public class KeyShell extends Configured
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. Nothing\n"
- + "was deleted. Use the --provider option to specify a provider.");
+ + "was deleted. Use the -provider option to specify a provider.");
return false;
}
if (keyName == null) {
out.println("There is no keyName specified. Please specify a " +
- "<keyname>. See the usage description with --help.");
+ "<keyname>. See the usage description with -help.");
return false;
}
if (interactive) {
@@ -436,19 +436,19 @@ public class KeyShell extends Configured
private class CreateCommand extends Command {
public static final String USAGE =
- "create <keyname> [--cipher <cipher>] [--size <size>]\n" +
- " [--description <description>]\n" +
- " [--attr <attribute=value>]\n" +
- " [--provider <provider>] [--help]";
+ "create <keyname> [-cipher <cipher>] [-size <size>]\n" +
+ " [-description <description>]\n" +
+ " [-attr <attribute=value>]\n" +
+ " [-provider <provider>] [-help]";
public static final String DESC =
"The create subcommand creates a new key for the name specified\n" +
"by the <keyname> argument within the provider specified by the\n" +
- "--provider argument. You may specify a cipher with the --cipher\n" +
+ "-provider argument. You may specify a cipher with the -cipher\n" +
"argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" +
- "The default keysize is 256. You may specify the requested key\n" +
- "length using the --size argument. Arbitrary attribute=value\n" +
- "style attributes may be specified using the --attr argument.\n" +
- "--attr may be specified multiple times, once per attribute.\n";
+ "The default keysize is 128. You may specify the requested key\n" +
+ "length using the -size argument. Arbitrary attribute=value\n" +
+ "style attributes may be specified using the -attr argument.\n" +
+ "-attr may be specified multiple times, once per attribute.\n";
final String keyName;
final Options options;
@@ -463,13 +463,13 @@ public class KeyShell extends Configured
provider = getKeyProvider();
if (provider == null) {
out.println("There are no valid KeyProviders configured. No key\n" +
- " was created. You can use the --provider option to specify\n" +
+ " was created. You can use the -provider option to specify\n" +
" a provider to use.");
rc = false;
}
if (keyName == null) {
out.println("Please provide a <keyname>. See the usage description" +
- " with --help.");
+ " with -help.");
rc = false;
}
return rc;
@@ -479,7 +479,8 @@ public class KeyShell extends Configured
warnIfTransientProvider();
try {
provider.createKey(keyName, options);
- out.println(keyName + " has been successfully created.");
+ out.println(keyName + " has been successfully created with options "
+ + options.toString() + ".");
provider.flush();
printProviderWritten();
} catch (InvalidParameterException e) {
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Wed Aug 20 01:34:29 2014
@@ -22,15 +22,18 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
-import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.http.client.utils.URIBuilder;
import org.codehaus.jackson.map.ObjectMapper;
@@ -50,6 +53,7 @@ import java.net.URL;
import java.net.URLEncoder;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
+import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -69,7 +73,10 @@ import com.google.common.base.Preconditi
* KMS client <code>KeyProvider</code> implementation.
*/
@InterfaceAudience.Private
-public class KMSClientProvider extends KeyProvider implements CryptoExtension {
+public class KMSClientProvider extends KeyProvider implements CryptoExtension,
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension {
+
+ public static final String TOKEN_KIND = "kms-dt";
public static final String SCHEME_NAME = "kms";
@@ -229,6 +236,8 @@ public class KMSClientProvider extends K
private String kmsUrl;
private SSLFactory sslFactory;
private ConnectionConfigurator configurator;
+ private DelegationTokenAuthenticatedURL.Token authToken;
+ private UserGroupInformation loginUgi;
@Override
public String toString() {
@@ -309,6 +318,8 @@ public class KMSClientProvider extends K
CommonConfigurationKeysPublic.
KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
new EncryptedQueueRefiller());
+ authToken = new DelegationTokenAuthenticatedURL.Token();
+ loginUgi = UserGroupInformation.getCurrentUser();
}
private String createServiceURL(URL url) throws IOException {
@@ -325,12 +336,14 @@ public class KMSClientProvider extends K
try {
StringBuilder sb = new StringBuilder();
sb.append(kmsUrl);
- sb.append(collection);
- if (resource != null) {
- sb.append("/").append(URLEncoder.encode(resource, UTF8));
- }
- if (subResource != null) {
- sb.append("/").append(subResource);
+ if (collection != null) {
+ sb.append(collection);
+ if (resource != null) {
+ sb.append("/").append(URLEncoder.encode(resource, UTF8));
+ if (subResource != null) {
+ sb.append("/").append(subResource);
+ }
+ }
}
URIBuilder uriBuilder = new URIBuilder(sb.toString());
if (parameters != null) {
@@ -365,14 +378,29 @@ public class KMSClientProvider extends K
return conn;
}
- private HttpURLConnection createConnection(URL url, String method)
+ private HttpURLConnection createConnection(final URL url, String method)
throws IOException {
HttpURLConnection conn;
try {
- AuthenticatedURL authUrl = new AuthenticatedURL(new PseudoAuthenticator(),
- configurator);
- conn = authUrl.openConnection(url, new AuthenticatedURL.Token());
- } catch (AuthenticationException ex) {
+ // if current UGI is different from UGI at constructor time, behave as
+ // proxyuser
+ UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
+ final String doAsUser =
+ (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
+ ? null : currentUgi.getShortUserName();
+
+ // creating the HTTP connection using the current UGI at constructor time
+ conn = loginUgi.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
+ @Override
+ public HttpURLConnection run() throws Exception {
+ DelegationTokenAuthenticatedURL authUrl =
+ new DelegationTokenAuthenticatedURL(configurator);
+ return authUrl.openConnection(url, authToken, doAsUser);
+ }
+ });
+ } catch (IOException ex) {
+ throw ex;
+ } catch (Exception ex) {
throw new IOException(ex);
}
conn.setUseCaches(false);
@@ -403,20 +431,27 @@ public class KMSClientProvider extends K
if (status != expected) {
InputStream es = null;
try {
- es = conn.getErrorStream();
- ObjectMapper mapper = new ObjectMapper();
- Map json = mapper.readValue(es, Map.class);
- String exClass = (String) json.get(
- KMSRESTConstants.ERROR_EXCEPTION_JSON);
- String exMsg = (String)
- json.get(KMSRESTConstants.ERROR_MESSAGE_JSON);
Exception toThrow;
- try {
- ClassLoader cl = KMSClientProvider.class.getClassLoader();
- Class klass = cl.loadClass(exClass);
- Constructor constr = klass.getConstructor(String.class);
- toThrow = (Exception) constr.newInstance(exMsg);
- } catch (Exception ex) {
+ String contentType = conn.getHeaderField(CONTENT_TYPE);
+ if (contentType != null &&
+ contentType.toLowerCase().startsWith(APPLICATION_JSON_MIME)) {
+ es = conn.getErrorStream();
+ ObjectMapper mapper = new ObjectMapper();
+ Map json = mapper.readValue(es, Map.class);
+ String exClass = (String) json.get(
+ KMSRESTConstants.ERROR_EXCEPTION_JSON);
+ String exMsg = (String)
+ json.get(KMSRESTConstants.ERROR_MESSAGE_JSON);
+ try {
+ ClassLoader cl = KMSClientProvider.class.getClassLoader();
+ Class klass = cl.loadClass(exClass);
+ Constructor constr = klass.getConstructor(String.class);
+ toThrow = (Exception) constr.newInstance(exMsg);
+ } catch (Exception ex) {
+ toThrow = new IOException(MessageFormat.format(
+ "HTTP status [{0}], {1}", status, conn.getResponseMessage()));
+ }
+ } else {
toThrow = new IOException(MessageFormat.format(
"HTTP status [{0}], {1}", status, conn.getResponseMessage()));
}
@@ -512,7 +547,7 @@ public class KMSClientProvider extends K
List<String> batch = new ArrayList<String>();
int batchLen = 0;
for (String name : keyNames) {
- int additionalLen = KMSRESTConstants.KEY_OP.length() + 1 + name.length();
+ int additionalLen = KMSRESTConstants.KEY.length() + 1 + name.length();
batchLen += additionalLen;
// topping at 1500 to account for initial URL and encoded names
if (batchLen > 1500) {
@@ -536,7 +571,7 @@ public class KMSClientProvider extends K
for (String[] keySet : keySets) {
if (keyNames.length > 0) {
Map<String, Object> queryStr = new HashMap<String, Object>();
- queryStr.put(KMSRESTConstants.KEY_OP, keySet);
+ queryStr.put(KMSRESTConstants.KEY, keySet);
URL url = createURL(KMSRESTConstants.KEYS_METADATA_RESOURCE, null,
null, queryStr);
HttpURLConnection conn = createConnection(url, HTTP_GET);
@@ -653,7 +688,7 @@ public class KMSClientProvider extends K
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', is '%s'",
- KeyProviderCryptoExtension.EK,
+ KeyProviderCryptoExtension.EEK,
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
);
checkNotNull(encryptedKeyVersion.getEncryptedKeyVersion(), "encryptedKey");
@@ -729,4 +764,25 @@ public class KMSClientProvider extends K
}
}
+ @Override
+ public Token<?>[] addDelegationTokens(String renewer,
+ Credentials credentials) throws IOException {
+ Token<?>[] tokens;
+ URL url = createURL(null, null, null, null);
+ DelegationTokenAuthenticatedURL authUrl =
+ new DelegationTokenAuthenticatedURL(configurator);
+ try {
+ Token<?> token = authUrl.getDelegationToken(url, authToken, renewer);
+ if (token != null) {
+ credentials.addToken(token.getService(), token);
+ tokens = new Token<?>[] { token };
+ } else {
+ throw new IOException("Got NULL as delegation token");
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException(ex);
+ }
+ return tokens;
+ }
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java Wed Aug 20 01:34:29 2014
@@ -37,7 +37,7 @@ public class KMSRESTConstants {
public static final String EEK_SUB_RESOURCE = "_eek";
public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion";
- public static final String KEY_OP = "key";
+ public static final String KEY = "key";
public static final String EEK_OP = "eek_op";
public static final String EEK_GENERATE = "generate";
public static final String EEK_DECRYPT = "decrypt";
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java Wed Aug 20 01:34:29 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Options.Crea
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.security.AccessControlException;
@@ -805,6 +806,18 @@ public abstract class AbstractFileSystem
/**
* The specification of this method matches that of
+ * {@link FileContext#access(Path, FsAction)}
+ * except that an UnresolvedLinkException may be thrown if a symlink is
+ * encountered in the path.
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ FileSystem.checkAccessPermissions(this.getFileStatus(path), mode);
+ }
+
+ /**
+ * The specification of this method matches that of
* {@link FileContext#getFileLinkStatus(Path)}
* except that an UnresolvedLinkException may be thrown if a symlink is
* encountered in the path leading up to the final path component.
@@ -1040,21 +1053,10 @@ public abstract class AbstractFileSystem
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -1069,21 +1071,10 @@ public abstract class AbstractFileSystem
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -1099,18 +1090,10 @@ public abstract class AbstractFileSystem
/**
* Get an xattr for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * An xattr will only be returned when the logged-in user has the correct permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -1127,13 +1110,7 @@ public abstract class AbstractFileSystem
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -1149,13 +1126,7 @@ public abstract class AbstractFileSystem
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -1173,14 +1144,7 @@ public abstract class AbstractFileSystem
* Only the xattr names for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for the "user" and "trusted"
- * namespaces.
- * The xattr names in the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -1194,21 +1158,10 @@ public abstract class AbstractFileSystem
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Wed Aug 20 01:34:29 2014
@@ -134,6 +134,9 @@ public class CommonConfigurationKeys ext
HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL =
"security.service.authorization.default.acl";
public static final String
+ HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_BLOCKED_ACL =
+ "security.service.authorization.default.acl.blocked";
+ public static final String
HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY =
"security.refresh.policy.protocol.acl";
public static final String
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Wed Aug 20 01:34:29 2014
@@ -207,7 +207,7 @@ public class CommonConfigurationKeysPubl
public static final String IPC_CLIENT_TCPNODELAY_KEY =
"ipc.client.tcpnodelay";
/** Defalt value for IPC_CLIENT_TCPNODELAY_KEY */
- public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = false;
+ public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = true;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IPC_SERVER_LISTEN_QUEUE_SIZE_KEY =
"ipc.server.listen.queue.size";
@@ -226,7 +226,7 @@ public class CommonConfigurationKeysPubl
public static final String IPC_SERVER_TCPNODELAY_KEY =
"ipc.server.tcpnodelay";
/** Default value for IPC_SERVER_TCPNODELAY_KEY */
- public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = false;
+ public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = true;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY =
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java Wed Aug 20 01:34:29 2014
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
/** Store the summary of a content (a directory or a file). */
@InterfaceAudience.Public
@@ -102,7 +103,7 @@ public class ContentSummary implements W
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
*/
- private static final String STRING_FORMAT = "%12d %12d %18d ";
+ private static final String STRING_FORMAT = "%12s %12s %18s ";
/**
* Output format:
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
@@ -117,7 +118,7 @@ public class ContentSummary implements W
private static final String QUOTA_HEADER = String.format(
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
- "quota", "remaining quota", "space quota", "reamaining quota") +
+ "name quota", "rem name quota", "space quota", "rem space quota") +
HEADER;
/** Return the header of the output.
@@ -139,11 +140,25 @@ public class ContentSummary implements W
/** Return the string representation of the object in the output format.
* if qOption is false, output directory count, file count, and content size;
* if qOption is true, output quota and remaining quota as well.
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption) {
+ return toString(qOption, false);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if qOption is false, output directory count, file count, and content size;
+ * if qOption is true, output quota and remaining quota as well.
+ * if hOption is false file sizes are returned in bytes
+ * if hOption is true file sizes are returned in human readable
*
* @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output if to be used
* @return the string representation of the object
*/
- public String toString(boolean qOption) {
+ public String toString(boolean qOption, boolean hOption) {
String prefix = "";
if (qOption) {
String quotaStr = "none";
@@ -152,19 +167,32 @@ public class ContentSummary implements W
String spaceQuotaRem = "inf";
if (quota>0) {
- quotaStr = Long.toString(quota);
- quotaRem = Long.toString(quota-(directoryCount+fileCount));
+ quotaStr = formatSize(quota, hOption);
+ quotaRem = formatSize(quota-(directoryCount+fileCount), hOption);
}
if (spaceQuota>0) {
- spaceQuotaStr = Long.toString(spaceQuota);
- spaceQuotaRem = Long.toString(spaceQuota - spaceConsumed);
+ spaceQuotaStr = formatSize(spaceQuota, hOption);
+ spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
}
- return prefix + String.format(STRING_FORMAT, directoryCount,
- fileCount, length);
+ return prefix + String.format(STRING_FORMAT,
+ formatSize(directoryCount, hOption),
+ formatSize(fileCount, hOption),
+ formatSize(length, hOption));
+ }
+ /**
+ * Formats a size to be human readable or in bytes
+ * @param size value to be formatted
+ * @param humanReadable flag indicating human readable or not
+ * @return String representation of the size
+ */
+ private String formatSize(long size, boolean humanReadable) {
+ return humanReadable
+ ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
+ : String.valueOf(size);
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Wed Aug 20 01:34:29 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileSystem.S
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -1109,6 +1110,55 @@ public final class FileContext {
}
/**
+ * Checks if the user can access a path. The mode specifies which access
+ * checks to perform. If the requested permissions are granted, then the
+ * method returns normally. If access is denied, then the method throws an
+ * {@link AccessControlException}.
+ * <p/>
+ * The default implementation of this method calls {@link #getFileStatus(Path)}
+ * and checks the returned permissions against the requested permissions.
+ * Note that the getFileStatus call will be subject to authorization checks.
+ * Typically, this requires search (execute) permissions on each directory in
+ * the path's prefix, but this is implementation-defined. Any file system
+ * that provides a richer authorization model (such as ACLs) may override the
+ * default implementation so that it checks against that model instead.
+ * <p>
+ * In general, applications should avoid using this method, due to the risk of
+ * time-of-check/time-of-use race conditions. The permissions on a file may
+ * change immediately after the access call returns. Most applications should
+ * prefer running specific file system actions as the desired user represented
+ * by a {@link UserGroupInformation}.
+ *
+ * @param path Path to check
+ * @param mode type of access to check
+ * @throws AccessControlException if access is denied
+ * @throws FileNotFoundException if the path does not exist
+ * @throws UnsupportedFileSystemException if file system for <code>path</code>
+ * is not supported
+ * @throws IOException see specific implementation
+ *
+ * Exceptions applicable to file systems accessed over RPC:
+ * @throws RpcClientException If an exception occurred in the RPC client
+ * @throws RpcServerException If an exception occurred in the RPC server
+ * @throws UnexpectedServerException If server implementation throws
+ * undeclared exception to RPC server
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(final Path path, final FsAction mode)
+ throws AccessControlException, FileNotFoundException,
+ UnsupportedFileSystemException, IOException {
+ final Path absPath = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(AbstractFileSystem fs, Path p) throws IOException,
+ UnresolvedLinkException {
+ fs.access(p, mode);
+ return null;
+ }
+ }.resolve(this, absPath);
+ }
+
+ /**
* Return a file status object that represents the path. If the path
* refers to a symlink then the FileStatus of the symlink is returned.
* The behavior is equivalent to #getFileStatus() if the underlying
@@ -2297,21 +2347,10 @@ public final class FileContext {
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2326,21 +2365,10 @@ public final class FileContext {
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2363,19 +2391,10 @@ public final class FileContext {
/**
* Get an xattr for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- *
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * An xattr will only be returned when the logged-in user has the correct permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -2398,13 +2417,7 @@ public final class FileContext {
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -2426,13 +2439,7 @@ public final class FileContext {
* Only those xattrs for which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattr of "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -2453,21 +2460,10 @@ public final class FileContext {
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
@@ -2490,14 +2486,7 @@ public final class FileContext {
* Only those xattr names which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for "user" and "trusted"
- * namespaces.
- * The xattrs of the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return List<String> of the XAttr names of the file or directory
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Wed Aug 20 01:34:29 2014
@@ -25,6 +25,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -50,6 +51,7 @@ import org.apache.hadoop.fs.Options.Chec
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
@@ -2073,6 +2075,71 @@ public abstract class FileSystem extends
public abstract FileStatus getFileStatus(Path f) throws IOException;
/**
+ * Checks if the user can access a path. The mode specifies which access
+ * checks to perform. If the requested permissions are granted, then the
+ * method returns normally. If access is denied, then the method throws an
+ * {@link AccessControlException}.
+ * <p/>
+ * The default implementation of this method calls {@link #getFileStatus(Path)}
+ * and checks the returned permissions against the requested permissions.
+ * Note that the getFileStatus call will be subject to authorization checks.
+ * Typically, this requires search (execute) permissions on each directory in
+ * the path's prefix, but this is implementation-defined. Any file system
+ * that provides a richer authorization model (such as ACLs) may override the
+ * default implementation so that it checks against that model instead.
+ * <p>
+ * In general, applications should avoid using this method, due to the risk of
+ * time-of-check/time-of-use race conditions. The permissions on a file may
+ * change immediately after the access call returns. Most applications should
+ * prefer running specific file system actions as the desired user represented
+ * by a {@link UserGroupInformation}.
+ *
+ * @param path Path to check
+ * @param mode type of access to check
+ * @throws AccessControlException if access is denied
+ * @throws FileNotFoundException if the path does not exist
+ * @throws IOException see specific implementation
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ checkAccessPermissions(this.getFileStatus(path), mode);
+ }
+
+ /**
+ * This method provides the default implementation of
+ * {@link #access(Path, FsAction)}.
+ *
+ * @param stat FileStatus to check
+ * @param mode type of access to check
+ * @throws IOException for any error
+ */
+ @InterfaceAudience.Private
+ static void checkAccessPermissions(FileStatus stat, FsAction mode)
+ throws IOException {
+ FsPermission perm = stat.getPermission();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String user = ugi.getShortUserName();
+ List<String> groups = Arrays.asList(ugi.getGroupNames());
+ if (user.equals(stat.getOwner())) {
+ if (perm.getUserAction().implies(mode)) {
+ return;
+ }
+ } else if (groups.contains(stat.getGroup())) {
+ if (perm.getGroupAction().implies(mode)) {
+ return;
+ }
+ } else {
+ if (perm.getOtherAction().implies(mode)) {
+ return;
+ }
+ }
+ throw new AccessControlException(String.format(
+ "Permission denied: user=%s, path=\"%s\":%s:%s:%s%s", user, stat.getPath(),
+ stat.getOwner(), stat.getGroup(), stat.isDirectory() ? "d" : "-", perm));
+ }
+
+ /**
* See {@link FileContext#fixRelativePart}
*/
protected Path fixRelativePart(Path p) {
@@ -2364,21 +2431,10 @@ public abstract class FileSystem extends
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2393,21 +2449,10 @@ public abstract class FileSystem extends
/**
* Set an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only set an xattr for the "user" namespace.
- * The super user can set an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set if the logged-in user has the correct permissions.
- * If the xattr exists, it is replaced.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to modify
* @param name xattr name.
@@ -2423,20 +2468,10 @@ public abstract class FileSystem extends
/**
* Get an xattr name and value for a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- *
- * A regular user can only get an xattr for the "user" namespace.
- * The super user can get an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * An xattr will only be returned if the logged-in user has the
- * correct permissions.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attribute
* @param name xattr name.
@@ -2453,13 +2488,7 @@ public abstract class FileSystem extends
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return Map<String, byte[]> describing the XAttrs of the file or directory
@@ -2475,13 +2504,7 @@ public abstract class FileSystem extends
* Only those xattrs which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattrs for the "user" namespace.
- * The super user can only get xattrs for "user" and "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @param names XAttr names.
@@ -2499,14 +2522,7 @@ public abstract class FileSystem extends
* Only those xattr names which the logged-in user has permissions to view
* are returned.
* <p/>
- * A regular user can only get xattr names for the "user" namespace.
- * The super user can only get xattr names for "user" and "trusted"
- * namespaces.
- * The xattrs of the "security" and "system" namespaces are only
- * used/exposed internally by/to the FS impl.
- * <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
* @return List<String> of the XAttr names of the file or directory
@@ -2519,21 +2535,10 @@ public abstract class FileSystem extends
/**
* Remove an xattr of a file or directory.
- * The name must be prefixed with user/trusted/security/system and
- * followed by ".". For example, "user.attr".
- * <p/>
- * A regular user can only remove an xattr for the "user" namespace.
- * The super user can remove an xattr of either the "user" or "trusted" namespaces.
- * The xattrs of the "security" and "system" namespaces are only used/exposed
- * internally by/to the FS impl.
- * <p/>
- * The access permissions of an xattr in the "user" namespace are
- * defined by the file and directory permission bits.
- * An xattr can only be set when the logged-in user has the correct permissions.
- * If the xattr exists, it will be replaced.
+ * The name must be prefixed with the namespace followed by ".". For example,
+ * "user.attr".
* <p/>
- * @see <a href="http://en.wikipedia.org/wiki/Extended_file_attributes">
- * http://en.wikipedia.org/wiki/Extended_file_attributes</a>
+ * Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to remove extended attribute
* @param name xattr name
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java Wed Aug 20 01:34:29 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
@@ -397,6 +398,12 @@ public class FilterFileSystem extends Fi
return fs.getFileStatus(f);
}
+ @Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ fs.access(path, mode);
+ }
+
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java Wed Aug 20 01:34:29 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.security.AccessControlException;
@@ -120,6 +121,13 @@ public abstract class FilterFs extends A
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ checkPath(path);
+ myFs.access(path, mode);
+ }
+
+ @Override
public FileStatus getFileLinkStatus(final Path f)
throws IOException, UnresolvedLinkException {
checkPath(f);
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java Wed Aug 20 01:34:29 2014
@@ -42,16 +42,22 @@ public class Count extends FsCommand {
factory.addClass(Count.class, "-count");
}
+ private static final String OPTION_QUOTA = "q";
+ private static final String OPTION_HUMAN = "h";
+
public static final String NAME = "count";
- public static final String USAGE = "[-q] <path> ...";
+ public static final String USAGE =
+ "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
- " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME";
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
+ "The -h option shows file sizes in human readable format.";
private boolean showQuotas;
+ private boolean humanReadable;
/** Constructor */
public Count() {}
@@ -70,17 +76,37 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
- CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "q");
+ CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
+ OPTION_QUOTA, OPTION_HUMAN);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
args.add(".");
}
- showQuotas = cf.getOpt("q");
+ showQuotas = cf.getOpt(OPTION_QUOTA);
+ humanReadable = cf.getOpt(OPTION_HUMAN);
}
@Override
protected void processPath(PathData src) throws IOException {
ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas) + src);
+ out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ }
+
+ /**
+ * Should quotas get shown as part of the report?
+ * @return if quotas should be shown then true otherwise false
+ */
+ @InterfaceAudience.Private
+ boolean isShowQuotas() {
+ return showQuotas;
+ }
+
+ /**
+ * Should sizes be shown in human readable format rather than bytes?
+ * @return true if human readable format
+ */
+ @InterfaceAudience.Private
+ boolean isHumanReadable() {
+ return humanReadable;
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java Wed Aug 20 01:34:29 2014
@@ -118,7 +118,11 @@ class Delete {
} catch(FileNotFoundException fnfe) {
throw fnfe;
} catch (IOException ioe) {
- throw new IOException(ioe.getMessage() + ". Consider using -skipTrash option", ioe);
+ String msg = ioe.getMessage();
+ if (ioe.getCause() != null) {
+ msg += ": " + ioe.getCause().getMessage();
+ }
+ throw new IOException(msg + ". Consider using -skipTrash option", ioe);
}
}
return success;
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java Wed Aug 20 01:34:29 2014
@@ -41,7 +41,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
/**
@@ -223,6 +225,12 @@ class ChRootedFileSystem extends FilterF
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ super.access(fullPath(path), mode);
+ }
+
+ @Override
public FsStatus getStatus(Path p) throws IOException {
return super.getStatus(fullPath(p));
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java Wed Aug 20 01:34:29 2014
@@ -41,7 +41,9 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -200,6 +202,11 @@ class ChRootedFs extends AbstractFileSys
return myFs.getFileStatus(fullPath(f));
}
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ myFs.access(fullPath(path), mode);
+ }
+
@Override
public FileStatus getFileLinkStatus(final Path f)
throws IOException, UnresolvedLinkException {
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java Wed Aug 20 01:34:29 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.fs.XAttrSetFlag
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
@@ -359,7 +360,14 @@ public class ViewFileSystem extends File
return new ViewFsFileStatus(status, this.makeQualified(f));
}
-
+ @Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, IOException {
+ InodeTree.ResolveResult<FileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.access(res.remainingPath, mode);
+ }
+
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
FileNotFoundException, IOException {
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java Wed Aug 20 01:34:29 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.local.LocalC
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
@@ -353,6 +354,14 @@ public class ViewFs extends AbstractFile
}
@Override
+ public void access(Path path, FsAction mode) throws AccessControlException,
+ FileNotFoundException, UnresolvedLinkException, IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.access(res.remainingPath, mode);
+ }
+
+ @Override
public FileStatus getFileLinkStatus(final Path f)
throws AccessControlException, FileNotFoundException,
UnsupportedFileSystemException, IOException {
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java Wed Aug 20 01:34:29 2014
@@ -1005,7 +1005,7 @@ public final class HttpServer2 implement
String remoteUser = request.getRemoteUser();
if (remoteUser == null) {
- response.sendError(HttpServletResponse.SC_UNAUTHORIZED,
+ response.sendError(HttpServletResponse.SC_FORBIDDEN,
"Unauthenticated users are not " +
"authorized to access this page.");
return false;
@@ -1013,7 +1013,7 @@ public final class HttpServer2 implement
if (servletContext.getAttribute(ADMINS_ACL) != null &&
!userHasAdministratorAccess(servletContext, remoteUser)) {
- response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "User "
+ response.sendError(HttpServletResponse.SC_FORBIDDEN, "User "
+ remoteUser + " is unauthorized to access this page.");
return false;
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java Wed Aug 20 01:34:29 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
@@ -823,6 +824,14 @@ public class NativeIO {
}
}
+ public static void link(File src, File dst) throws IOException {
+ if (!nativeLoaded) {
+ HardLink.createHardLink(src, dst);
+ } else {
+ link0(src.getAbsolutePath(), dst.getAbsolutePath());
+ }
+ }
+
/**
* A version of renameTo that throws a descriptive exception when it fails.
*
@@ -833,4 +842,7 @@ public class NativeIO {
*/
private static native void renameTo0(String src, String dst)
throws NativeIOException;
+
+ private static native void link0(String src, String dst)
+ throws NativeIOException;
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java Wed Aug 20 01:34:29 2014
@@ -88,13 +88,13 @@ public class RpcMetrics {
@Metric("Processsing time") MutableRate rpcProcessingTime;
MutableQuantiles[] rpcProcessingTimeMillisQuantiles;
@Metric("Number of authentication failures")
- MutableCounterInt rpcAuthenticationFailures;
+ MutableCounterLong rpcAuthenticationFailures;
@Metric("Number of authentication successes")
- MutableCounterInt rpcAuthenticationSuccesses;
+ MutableCounterLong rpcAuthenticationSuccesses;
@Metric("Number of authorization failures")
- MutableCounterInt rpcAuthorizationFailures;
+ MutableCounterLong rpcAuthorizationFailures;
@Metric("Number of authorization sucesses")
- MutableCounterInt rpcAuthorizationSuccesses;
+ MutableCounterLong rpcAuthorizationSuccesses;
@Metric("Number of open connections") public int numOpenConnections() {
return server.getNumOpenConnections();
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java Wed Aug 20 01:34:29 2014
@@ -143,6 +143,12 @@ public class JMXJsonServlet extends Http
jsonFactory = new JsonFactory();
}
+ protected boolean isInstrumentationAccessAllowed(HttpServletRequest request,
+ HttpServletResponse response) throws IOException {
+ return HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
+ request, response);
+ }
+
/**
* Process a GET request for the specified resource.
*
@@ -154,8 +160,7 @@ public class JMXJsonServlet extends Http
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
try {
- if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
- request, response)) {
+ if (!isInstrumentationAccessAllowed(request, response)) {
return;
}
JsonGenerator jg = null;
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsCollectorImpl.java Wed Aug 20 01:34:29 2014
@@ -21,14 +21,18 @@ package org.apache.hadoop.metrics2.impl;
import java.util.Iterator;
import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsFilter;
import static org.apache.hadoop.metrics2.lib.Interns.*;
-class MetricsCollectorImpl implements MetricsCollector,
+@InterfaceAudience.Private
+@VisibleForTesting
+public class MetricsCollectorImpl implements MetricsCollector,
Iterable<MetricsRecordBuilderImpl> {
private final List<MetricsRecordBuilderImpl> rbs = Lists.newArrayList();
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java Wed Aug 20 01:34:29 2014
@@ -90,6 +90,14 @@ public class MutableStat extends Mutable
}
/**
+ * Set whether to display the extended stats (stdev, min/max etc.) or not
+ * @param extended enable/disable displaying extended stats
+ */
+ public synchronized void setExtended(boolean extended) {
+ this.extended = extended;
+ }
+
+ /**
* Add a number of samples and their sum to the running stat
* @param numSamples number of samples
* @param sum of the samples
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java Wed Aug 20 01:34:29 2014
@@ -293,7 +293,7 @@ public class NetworkTopologyWithNodeGrou
return;
}
}
- super.sortByDistance(reader, nodes, nodes.length, seed,
+ super.sortByDistance(reader, nodes, activeLen, seed,
randomizeBlockLocationsPerBlock);
}
Modified: hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java Wed Aug 20 01:34:29 2014
@@ -312,15 +312,15 @@ public class LdapGroupsMapping
useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
keystore = conf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
- keystorePass =
- conf.get(LDAP_KEYSTORE_PASSWORD_KEY, LDAP_KEYSTORE_PASSWORD_DEFAULT);
+ keystorePass = getPassword(conf, LDAP_KEYSTORE_PASSWORD_KEY,
+ LDAP_KEYSTORE_PASSWORD_DEFAULT);
if (keystorePass.isEmpty()) {
keystorePass = extractPassword(conf.get(LDAP_KEYSTORE_PASSWORD_FILE_KEY,
LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT));
}
bindUser = conf.get(BIND_USER_KEY, BIND_USER_DEFAULT);
- bindPassword = conf.get(BIND_PASSWORD_KEY, BIND_PASSWORD_DEFAULT);
+ bindPassword = getPassword(conf, BIND_PASSWORD_KEY, BIND_PASSWORD_DEFAULT);
if (bindPassword.isEmpty()) {
bindPassword = extractPassword(
conf.get(BIND_PASSWORD_FILE_KEY, BIND_PASSWORD_FILE_DEFAULT));
@@ -341,7 +341,25 @@ public class LdapGroupsMapping
this.conf = conf;
}
-
+
+ String getPassword(Configuration conf, String alias, String defaultPass) {
+ String password = null;
+ try {
+ char[] passchars = conf.getPassword(alias);
+ if (passchars != null) {
+ password = new String(passchars);
+ }
+ else {
+ password = defaultPass;
+ }
+ }
+ catch (IOException ioe) {
+ LOG.warn("Exception while trying to password for alias " + alias + ": "
+ + ioe.getMessage());
+ }
+ return password;
+ }
+
String extractPassword(String pwFile) {
if (pwFile.isEmpty()) {
// If there is no password file defined, we'll assume that we should do