You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:11 UTC
svn commit: r1619012 [10/14] - in
/hadoop/common/branches/HADOOP-10388/hadoop-common-project: ./ hadoop-auth/
hadoop-auth/dev-support/
hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/
hadoop-auth/src/main/java/org/apache/hado...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c Tue Aug 19 23:49:39 2014
@@ -55,40 +55,23 @@ static void pipelined_crc32c(uint32_t *c
static int cached_cpu_supports_crc32; // initialized by constructor below
static uint32_t crc32c_hardware(uint32_t crc, const uint8_t* data, size_t length);
-int bulk_calculate_crc(const uint8_t *data, size_t data_len,
- uint32_t *sums, int checksum_type,
- int bytes_per_checksum) {
- uint32_t crc;
- crc_update_func_t crc_update_func;
-
- switch (checksum_type) {
- case CRC32_ZLIB_POLYNOMIAL:
- crc_update_func = crc32_zlib_sb8;
- break;
- case CRC32C_POLYNOMIAL:
- crc_update_func = crc32c_sb8;
- break;
- default:
- return -EINVAL;
- break;
+static inline int store_or_verify(uint32_t *sums, uint32_t crc,
+ int is_verify) {
+ if (!is_verify) {
+ *sums = crc;
+ return 1;
+ } else {
+ return crc == *sums;
}
- while (likely(data_len > 0)) {
- int len = likely(data_len >= bytes_per_checksum) ? bytes_per_checksum : data_len;
- crc = CRC_INITIAL_VAL;
- crc = crc_update_func(crc, data, len);
- *sums = ntohl(crc_val(crc));
- data += len;
- data_len -= len;
- sums++;
- }
- return 0;
}
-int bulk_verify_crc(const uint8_t *data, size_t data_len,
- const uint32_t *sums, int checksum_type,
+int bulk_crc(const uint8_t *data, size_t data_len,
+ uint32_t *sums, int checksum_type,
int bytes_per_checksum,
crc32_error_t *error_info) {
+ int is_verify = error_info != NULL;
+
#ifdef USE_PIPELINED
uint32_t crc1, crc2, crc3;
int n_blocks = data_len / bytes_per_checksum;
@@ -112,7 +95,7 @@ int bulk_verify_crc(const uint8_t *data,
}
break;
default:
- return INVALID_CHECKSUM_TYPE;
+ return is_verify ? INVALID_CHECKSUM_TYPE : -EINVAL;
}
#ifdef USE_PIPELINED
@@ -122,16 +105,15 @@ int bulk_verify_crc(const uint8_t *data,
crc1 = crc2 = crc3 = CRC_INITIAL_VAL;
pipelined_crc32c(&crc1, &crc2, &crc3, data, bytes_per_checksum, 3);
- crc = ntohl(crc_val(crc1));
- if ((crc = ntohl(crc_val(crc1))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify)))
goto return_crc_error;
sums++;
data += bytes_per_checksum;
- if ((crc = ntohl(crc_val(crc2))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc2))), is_verify)))
goto return_crc_error;
sums++;
data += bytes_per_checksum;
- if ((crc = ntohl(crc_val(crc3))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc3))), is_verify)))
goto return_crc_error;
sums++;
data += bytes_per_checksum;
@@ -143,12 +125,12 @@ int bulk_verify_crc(const uint8_t *data,
crc1 = crc2 = crc3 = CRC_INITIAL_VAL;
pipelined_crc32c(&crc1, &crc2, &crc3, data, bytes_per_checksum, n_blocks);
- if ((crc = ntohl(crc_val(crc1))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify)))
goto return_crc_error;
data += bytes_per_checksum;
sums++;
if (n_blocks == 2) {
- if ((crc = ntohl(crc_val(crc2))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc2))), is_verify)))
goto return_crc_error;
sums++;
data += bytes_per_checksum;
@@ -160,10 +142,10 @@ int bulk_verify_crc(const uint8_t *data,
crc1 = crc2 = crc3 = CRC_INITIAL_VAL;
pipelined_crc32c(&crc1, &crc2, &crc3, data, remainder, 1);
- if ((crc = ntohl(crc_val(crc1))) != *sums)
+ if (unlikely(!store_or_verify(sums, (crc = ntohl(crc_val(crc1))), is_verify)))
goto return_crc_error;
}
- return CHECKSUMS_VALID;
+ return is_verify ? CHECKSUMS_VALID : 0;
}
#endif
@@ -172,14 +154,14 @@ int bulk_verify_crc(const uint8_t *data,
crc = CRC_INITIAL_VAL;
crc = crc_update_func(crc, data, len);
crc = ntohl(crc_val(crc));
- if (unlikely(crc != *sums)) {
+ if (unlikely(!store_or_verify(sums, crc, is_verify))) {
goto return_crc_error;
}
data += len;
data_len -= len;
sums++;
}
- return CHECKSUMS_VALID;
+ return is_verify ? CHECKSUMS_VALID : 0;
return_crc_error:
if (error_info != NULL) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h Tue Aug 19 23:49:39 2014
@@ -42,49 +42,32 @@ typedef struct crc32_error {
/**
- * Verify a buffer of data which is checksummed in chunks
- * of bytes_per_checksum bytes. The checksums are each 32 bits
- * and are stored in sequential indexes of the 'sums' array.
+ * Either calculates checksums for or verifies a buffer of data.
+ * Checksums performed in chunks of bytes_per_checksum bytes. The checksums
+ * are each 32 bits and are stored in sequential indexes of the 'sums' array.
+ * Verification is done (sums is assumed to already contain the checksums)
+ * if error_info is non-null; otherwise calculation is done and checksums
+ * are stored into sums.
*
* @param data The data to checksum
* @param dataLen Length of the data buffer
- * @param sums (out param) buffer to write checksums into.
- * It must contain at least dataLen * 4 bytes.
+ * @param sums (out param) buffer to write checksums into or
+ * where checksums are already stored.
+ * It must contain at least
+ * ((dataLen - 1) / bytes_per_checksum + 1) * 4 bytes.
* @param checksum_type One of the CRC32 algorithm constants defined
* above
* @param bytes_per_checksum How many bytes of data to process per checksum.
- * @param error_info If non-NULL, will be filled in if an error
- * is detected
+ * @param error_info If non-NULL, verification will be performed and
+ * it will be filled in if an error
+ * is detected. Otherwise calculation is performed.
*
* @return 0 for success, non-zero for an error, result codes
- * for which are defined above
+ * for verification are defined above
*/
-extern int bulk_verify_crc(const uint8_t *data, size_t data_len,
- const uint32_t *sums, int checksum_type,
+extern int bulk_crc(const uint8_t *data, size_t data_len,
+ uint32_t *sums, int checksum_type,
int bytes_per_checksum,
crc32_error_t *error_info);
-/**
- * Calculate checksums for some data.
- *
- * The checksums are each 32 bits and are stored in sequential indexes of the
- * 'sums' array.
- *
- * This function is not (yet) optimized. It is provided for testing purposes
- * only.
- *
- * @param data The data to checksum
- * @param dataLen Length of the data buffer
- * @param sums (out param) buffer to write checksums into.
- * It must contain at least dataLen * 4 bytes.
- * @param checksum_type One of the CRC32 algorithm constants defined
- * above
- * @param bytesPerChecksum How many bytes of data to process per checksum.
- *
- * @return 0 for success, non-zero for an error
- */
-int bulk_calculate_crc(const uint8_t *data, size_t data_len,
- uint32_t *sums, int checksum_type,
- int bytes_per_checksum);
-
#endif
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c Tue Aug 19 23:49:39 2014
@@ -48,9 +48,9 @@ static int testBulkVerifyCrc(int dataLen
sums = calloc(sizeof(uint32_t),
(dataLen + bytesPerChecksum - 1) / bytesPerChecksum);
- EXPECT_ZERO(bulk_calculate_crc(data, dataLen, sums, crcType,
- bytesPerChecksum));
- EXPECT_ZERO(bulk_verify_crc(data, dataLen, sums, crcType,
+ EXPECT_ZERO(bulk_crc(data, dataLen, sums, crcType,
+ bytesPerChecksum, NULL));
+ EXPECT_ZERO(bulk_crc(data, dataLen, sums, crcType,
bytesPerChecksum, &errorData));
free(data);
free(sums);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory Tue Aug 19 23:49:39 2014
@@ -15,3 +15,4 @@
org.apache.hadoop.crypto.key.JavaKeyStoreProvider$Factory
org.apache.hadoop.crypto.key.UserProvider$Factory
+org.apache.hadoop.crypto.key.kms.KMSClientProvider$Factory
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Tue Aug 19 23:49:39 2014
@@ -94,6 +94,98 @@
</description>
</property>
+<!--
+=== Multiple group mapping providers configuration sample ===
+ This sample illustrates a typical use case for CompositeGroupsMapping where
+Hadoop authentication uses MIT Kerberos which trusts an AD realm. In this case, service
+principals such as hdfs, mapred, hbase, hive, oozie and etc can be placed in In MIT Kerberos,
+but end users are just from the trusted AD. For the service principals, ShellBasedUnixGroupsMapping
+provider can be used to query their groups for efficiency, and for end users, LdapGroupsMapping
+provider can be used. This avoids to add group entries in AD for service principals when only using
+LdapGroupsMapping provider.
+ In case multiple ADs are involved and trusted by the MIT Kerberos in this use case, LdapGroupsMapping
+provider can be used more times with different AD specific configurations. This sample also shows how
+to do that. Here are the necessary configurations.
+
+<property>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.CompositeGroupsMapping</value>
+ <description>
+ Class for user to group mapping (get groups for a given user) for ACL, which
+ makes use of other multiple providers to provide the service.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.providers</name>
+ <value>shell4services,ad4usersX,ad4usersY</value>
+ <description>
+ Comma separated of names of other providers to provide user to group mapping.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.providers.combined</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether groups from the providers are combined or not. The default value is true
+ If true, then all the providers will be tried to get groups and all the groups are combined to return as
+ the final results. Otherwise, providers are tried one by one in the configured list order, and if any
+ groups are retrieved from any provider, then the groups will be returned without trying the left ones.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.shell4services</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'shell4services'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.ad4usersX</name>
+ <value>org.apache.hadoop.security.LdapGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'ad4usersX'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.provider.ad4usersY</name>
+ <value>org.apache.hadoop.security.LdapGroupsMapping</value>
+ <description>
+ Class for group mapping provider named by 'ad4usersY'. The name can then be referenced
+ by hadoop.security.group.mapping.providers property.
+ </description>
+</property>
+
+<property>
+<name>hadoop.security.group.mapping.provider.ad4usersX.ldap.url</name>
+<value>ldap://ad-host-for-users-X:389</value>
+ <description>
+ ldap url for the provider named by 'ad4usersX'. Note this property comes from
+ 'hadoop.security.group.mapping.ldap.url'.
+ </description>
+</property>
+
+<property>
+<name>hadoop.security.group.mapping.provider.ad4usersY.ldap.url</name>
+<value>ldap://ad-host-for-users-Y:389</value>
+ <description>
+ ldap url for the provider named by 'ad4usersY'. Note this property comes from
+ 'hadoop.security.group.mapping.ldap.url'.
+ </description>
+</property>
+
+You also need to configure other properties like
+ hadoop.security.group.mapping.ldap.bind.password.file and etc.
+for ldap providers in the same way as above does.
+
+-->
+
<property>
<name>hadoop.security.groups.cache.secs</name>
<value>300</value>
@@ -106,6 +198,20 @@
</property>
<property>
+ <name>hadoop.security.groups.negative-cache.secs</name>
+ <value>30</value>
+ <description>
+ Expiration time for entries in the the negative user-to-group mapping
+ caching, in seconds. This is useful when invalid users are retrying
+ frequently. It is suggested to set a small value for this expiration, since
+ a transient error in group lookup could temporarily lock out a legitimate
+ user.
+
+ Set this to zero or negative value to disable negative user-to-group caching.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.groups.cache.warn.after.ms</name>
<value>5000</value>
<description>
@@ -576,6 +682,14 @@
</property>
<property>
+ <name>fs.s3n.server-side-encryption-algorithm</name>
+ <value></value>
+ <description>Specify a server-side encryption algorithm for S3.
+ The default is NULL, and the only other currently allowable value is AES256.
+ </description>
+</property>
+
+<property>
<name>io.seqfile.compress.blocksize</name>
<value>1000000</value>
<description>The minimum block size for compression in block compressed
@@ -693,27 +807,19 @@
</description>
</property>
-<property>
- <name>ipc.server.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the server. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
- </description>
-</property>
+<!-- Proxy Configuration -->
<property>
- <name>ipc.client.tcpnodelay</name>
- <value>false</value>
- <description>Turn on/off Nagle's algorithm for the TCP socket connection on
- the client. Setting to true disables the algorithm and may decrease latency
- with a cost of more/smaller packets.
+ <name>hadoop.security.impersonation.provider.class</name>
+ <value></value>
+ <description>A class which implements ImpersonationProvider interface, used to
+ authorize whether one user can impersonate a specific user.
+ If not specified, the DefaultImpersonationProvider will be used.
+ If a class is specified, then that class will be used to determine
+ the impersonation capability.
</description>
</property>
-
-<!-- Proxy Configuration -->
-
<property>
<name>hadoop.rpc.socket.factory.class.default</name>
<value>org.apache.hadoop.net.StandardSocketFactory</value>
@@ -1290,18 +1396,17 @@
</property>
<property>
- <name>nfs3.server.port</name>
- <value>2049</value>
- <description>
- Specify the port number used by Hadoop NFS.
- </description>
-</property>
-
-<property>
- <name>nfs3.mountd.port</name>
- <value>4242</value>
+ <name>nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
<description>
- Specify the port number used by Hadoop mount daemon.
+ By default, the export can be mounted by any client. The value string
+ contains machine name and access privilege, separated by whitespace
+ characters. The machine name format can be a single host, a Java regular
+ expression, or an IPv4 address. The access privilege uses rw or ro to
+ specify read/write or read-only access of the machines to exports. If the
+ access privilege is not provided, the default is read-only. Entries are separated by ";".
+ For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;".
+ Only the NFS gateway needs to restart after this property is updated.
</description>
</property>
@@ -1340,4 +1445,42 @@
true.
</description>
</property>
+<property>
+ <name>fs.har.impl.disable.cache</name>
+ <value>true</value>
+ <description>Don't cache 'har' filesystem instances.</description>
+</property>
+
+<!--- KMSClientProvider configurations -->
+<property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.size</name>
+ <value>500</value>
+ <description>
+ Size of the EncryptedKeyVersion cache Queue for each key
+ </description>
+</property>
+<property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.low-watermark</name>
+ <value>0.3f</value>
+ <description>
+ If size of the EncryptedKeyVersion cache Queue falls below the
+ low watermark, this cache queue will be scheduled for a refill
+ </description>
+</property>
+<property>
+ <name>hadoop.security.kms.client.encrypted.key.cache.num.refill.threads</name>
+ <value>2</value>
+ <description>
+ Number of threads to use for refilling depleted EncryptedKeyVersion
+ cache Queues
+ </description>
+</property>
+<property>
+ <name>"hadoop.security.kms.client.encrypted.key.cache.expiry</name>
+ <value>43200000</value>
+ <description>
+ Cache expiry time for a Key, after which the cache Queue for this
+ key will be dropped. Default = 12hrs
+ </description>
+</property>
</configuration>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Tue Aug 19 23:49:39 2014
@@ -226,24 +226,34 @@ Hadoop MapReduce Next Generation - Clust
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.address>>> | | |
| | <<<ResourceManager>>> host:port for clients to submit jobs. | |
-| | | <host:port> |
+| | | <host:port>\ |
+| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.scheduler.address>>> | | |
| | <<<ResourceManager>>> host:port for ApplicationMasters to talk to | |
| | Scheduler to obtain resources. | |
-| | | <host:port> |
+| | | <host:port>\ |
+| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.resource-tracker.address>>> | | |
| | <<<ResourceManager>>> host:port for NodeManagers. | |
-| | | <host:port> |
+| | | <host:port>\ |
+| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.admin.address>>> | | |
| | <<<ResourceManager>>> host:port for administrative commands. | |
-| | | <host:port> |
+| | | <host:port>\ |
+| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.webapp.address>>> | | |
| | <<<ResourceManager>>> web-ui host:port. | |
-| | | <host:port> |
+| | | <host:port>\ |
+| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
+*-------------------------+-------------------------+------------------------+
+| <<<yarn.resourcemanager.hostname>>> | | |
+| | <<<ResourceManager>>> host. | |
+| | | <host>\ |
+| | | Single hostname that can be set in place of setting all <<<yarn.resourcemanager*address>>> resources. Results in default ports for ResourceManager components. |
*-------------------------+-------------------------+------------------------+
| <<<yarn.resourcemanager.scheduler.class>>> | | |
| | <<<ResourceManager>>> Scheduler class. | |
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm Tue Aug 19 23:49:39 2014
@@ -81,88 +81,30 @@ User Commands
* <<<archive>>>
- Creates a hadoop archive. More information can be found at Hadoop
- Archives.
-
- Usage: <<<hadoop archive -archiveName NAME <src>* <dest> >>>
-
-*-------------------+-------------------------------------------------------+
-||COMMAND_OPTION || Description
-*-------------------+-------------------------------------------------------+
-| -archiveName NAME | Name of the archive to be created.
-*-------------------+-------------------------------------------------------+
-| src | Filesystem pathnames which work as usual with regular
- | expressions.
-*-------------------+-------------------------------------------------------+
-| dest | Destination directory which would contain the archive.
-*-------------------+-------------------------------------------------------+
+ Creates a hadoop archive. More information can be found at
+ {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html}
+ Hadoop Archives Guide}}.
* <<<distcp>>>
Copy file or directories recursively. More information can be found at
- Hadoop DistCp Guide.
-
- Usage: <<<hadoop distcp <srcurl> <desturl> >>>
-
-*-------------------+--------------------------------------------+
-||COMMAND_OPTION || Description
-*-------------------+--------------------------------------------+
-| srcurl | Source Url
-*-------------------+--------------------------------------------+
-| desturl | Destination Url
-*-------------------+--------------------------------------------+
+ {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html}
+ Hadoop DistCp Guide}}.
* <<<fs>>>
- Usage: <<<hadoop fs [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
-
- Deprecated, use <<<hdfs dfs>>> instead.
-
- Runs a generic filesystem user client.
-
- The various COMMAND_OPTIONS can be found at File System Shell Guide.
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#dfs}<<<hdfs dfs>>>}}
+ instead.
* <<<fsck>>>
- Runs a HDFS filesystem checking utility.
- See {{{../hadoop-hdfs/HdfsUserGuide.html#fsck}fsck}} for more info.
-
- Usage: <<<hadoop fsck [GENERIC_OPTIONS] <path> [-move | -delete | -openforwrite] [-files [-blocks [-locations | -racks]]]>>>
-
-*------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------+---------------------------------------------+
-| <path> | Start checking from this path.
-*------------------+---------------------------------------------+
-| -move | Move corrupted files to /lost+found
-*------------------+---------------------------------------------+
-| -delete | Delete corrupted files.
-*------------------+---------------------------------------------+
-| -openforwrite | Print out files opened for write.
-*------------------+---------------------------------------------+
-| -files | Print out files being checked.
-*------------------+---------------------------------------------+
-| -blocks | Print out block report.
-*------------------+---------------------------------------------+
-| -locations | Print out locations for every block.
-*------------------+---------------------------------------------+
-| -racks | Print out network topology for data-node locations.
-*------------------+---------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#fsck}<<<hdfs fsck>>>}}
+ instead.
* <<<fetchdt>>>
- Gets Delegation Token from a NameNode.
- See {{{../hadoop-hdfs/HdfsUserGuide.html#fetchdt}fetchdt}} for more info.
-
- Usage: <<<hadoop fetchdt [GENERIC_OPTIONS] [--webservice <namenode_http_addr>] <path> >>>
-
-*------------------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------------------+---------------------------------------------+
-| <fileName> | File name to store the token into.
-*------------------------------+---------------------------------------------+
-| --webservice <https_address> | use http protocol instead of RPC
-*------------------------------+---------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#fetchdt}
+ <<<hdfs fetchdt>>>}} instead.
* <<<jar>>>
@@ -179,103 +121,21 @@ User Commands
* <<<job>>>
- Command to interact with Map Reduce Jobs.
-
- Usage: <<<hadoop job [GENERIC_OPTIONS] [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobOutputDir>] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>]>>>
-
-*------------------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------------------+---------------------------------------------+
-| -submit <job-file> | Submits the job.
-*------------------------------+---------------------------------------------+
-| -status <job-id> | Prints the map and reduce completion
- | percentage and all job counters.
-*------------------------------+---------------------------------------------+
-| -counter <job-id> <group-name> <counter-name> | Prints the counter value.
-*------------------------------+---------------------------------------------+
-| -kill <job-id> | Kills the job.
-*------------------------------+---------------------------------------------+
-| -events <job-id> <from-event-#> <#-of-events> | Prints the events' details
- | received by jobtracker for the given range.
-*------------------------------+---------------------------------------------+
-| -history [all]<jobOutputDir> | Prints job details, failed and killed tip
- | details. More details about the job such as
- | successful tasks and task attempts made for
- | each task can be viewed by specifying the [all]
- | option.
-*------------------------------+---------------------------------------------+
-| -list [all] | Displays jobs which are yet to complete.
- | <<<-list all>>> displays all jobs.
-*------------------------------+---------------------------------------------+
-| -kill-task <task-id> | Kills the task. Killed tasks are NOT counted
- | against failed attempts.
-*------------------------------+---------------------------------------------+
-| -fail-task <task-id> | Fails the task. Failed tasks are counted
- | against failed attempts.
-*------------------------------+---------------------------------------------+
-| -set-priority <job-id> <priority> | Changes the priority of the job. Allowed
- | priority values are VERY_HIGH, HIGH, NORMAL,
- | LOW, VERY_LOW
-*------------------------------+---------------------------------------------+
+ Deprecated. Use
+ {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#job}
+ <<<mapred job>>>}} instead.
* <<<pipes>>>
- Runs a pipes job.
-
- Usage: <<<hadoop pipes [-conf <path>] [-jobconf <key=value>, <key=value>,
- ...] [-input <path>] [-output <path>] [-jar <jar file>] [-inputformat
- <class>] [-map <class>] [-partitioner <class>] [-reduce <class>] [-writer
- <class>] [-program <executable>] [-reduces <num>]>>>
-
-*----------------------------------------+------------------------------------+
-|| COMMAND_OPTION || Description
-*----------------------------------------+------------------------------------+
-| -conf <path> | Configuration for job
-*----------------------------------------+------------------------------------+
-| -jobconf <key=value>, <key=value>, ... | Add/override configuration for job
-*----------------------------------------+------------------------------------+
-| -input <path> | Input directory
-*----------------------------------------+------------------------------------+
-| -output <path> | Output directory
-*----------------------------------------+------------------------------------+
-| -jar <jar file> | Jar filename
-*----------------------------------------+------------------------------------+
-| -inputformat <class> | InputFormat class
-*----------------------------------------+------------------------------------+
-| -map <class> | Java Map class
-*----------------------------------------+------------------------------------+
-| -partitioner <class> | Java Partitioner
-*----------------------------------------+------------------------------------+
-| -reduce <class> | Java Reduce class
-*----------------------------------------+------------------------------------+
-| -writer <class> | Java RecordWriter
-*----------------------------------------+------------------------------------+
-| -program <executable> | Executable URI
-*----------------------------------------+------------------------------------+
-| -reduces <num> | Number of reduces
-*----------------------------------------+------------------------------------+
+ Deprecated. Use
+ {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#pipes}
+ <<<mapred pipes>>>}} instead.
* <<<queue>>>
- command to interact and view Job Queue information
-
- Usage: <<<hadoop queue [-list] | [-info <job-queue-name> [-showJobs]] | [-showacls]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -list | Gets list of Job Queues configured in the system.
- | Along with scheduling information associated with the job queues.
-*-----------------+-----------------------------------------------------------+
-| -info <job-queue-name> [-showJobs] | Displays the job queue information and
- | associated scheduling information of particular job queue.
- | If <<<-showJobs>>> options is present a list of jobs
- | submitted to the particular job queue is displayed.
-*-----------------+-----------------------------------------------------------+
-| -showacls | Displays the queue name and associated queue operations
- | allowed for the current user. The list consists of only
- | those queues to which the user has access.
-*-----------------+-----------------------------------------------------------+
+ Deprecated. Use
+ {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapredCommands.html#queue}
+ <<<mapred queue>>>}} instead.
* <<<version>>>
@@ -294,9 +154,24 @@ User Commands
* <<<classpath>>>
Prints the class path needed to get the Hadoop jar and the required
- libraries.
+ libraries. If called without arguments, then prints the classpath set up by
+ the command scripts, which is likely to contain wildcards in the classpath
+ entries. Additional options print the classpath after wildcard expansion or
+ write the classpath into the manifest of a jar file. The latter is useful in
+ environments where wildcards cannot be used and the expanded classpath exceeds
+ the maximum supported command line length.
- Usage: <<<hadoop classpath>>>
+ Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>>
+
+*-----------------+-----------------------------------------------------------+
+|| COMMAND_OPTION || Description
+*-----------------+-----------------------------------------------------------+
+| --glob | expand wildcards
+*-----------------+-----------------------------------------------------------+
+| --jar <path> | write classpath as manifest in jar named <path>
+*-----------------+-----------------------------------------------------------+
+| -h, --help | print help
+*-----------------+-----------------------------------------------------------+
Administration Commands
@@ -304,18 +179,8 @@ Administration Commands
* <<<balancer>>>
- Runs a cluster balancing utility. An administrator can simply press Ctrl-C
- to stop the rebalancing process. See
- {{{../hadoop-hdfs/HdfsUserGuide.html#Rebalancer}Rebalancer}} for more details.
-
- Usage: <<<hadoop balancer [-threshold <threshold>]>>>
-
-*------------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION | Description
-*------------------------+-----------------------------------------------------------+
-| -threshold <threshold> | Percentage of disk capacity. This overwrites the
- | default threshold.
-*------------------------+-----------------------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#balancer}
+ <<<hdfs balancer>>>}} instead.
* <<<daemonlog>>>
@@ -338,164 +203,20 @@ Administration Commands
* <<<datanode>>>
- Runs a HDFS datanode.
-
- Usage: <<<hadoop datanode [-rollback]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -rollback | Rollsback the datanode to the previous version. This should
- | be used after stopping the datanode and distributing the old
- | hadoop version.
-*-----------------+-----------------------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#datanode}
+ <<<hdfs datanode>>>}} instead.
* <<<dfsadmin>>>
- Runs a HDFS dfsadmin client.
-
- Usage: <<<hadoop dfsadmin [GENERIC_OPTIONS] [-report] [-safemode enter | leave | get | wait] [-refreshNodes] [-finalizeUpgrade] [-upgradeProgress status | details | force] [-metasave filename] [-setQuota <quota> <dirname>...<dirname>] [-clrQuota <dirname>...<dirname>] [-restoreFailedStorage true|false|check] [-help [cmd]]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -report | Reports basic filesystem information and statistics.
-*-----------------+-----------------------------------------------------------+
-| -safemode enter / leave / get / wait | Safe mode maintenance command. Safe
- | mode is a Namenode state in which it \
- | 1. does not accept changes to the name space (read-only) \
- | 2. does not replicate or delete blocks. \
- | Safe mode is entered automatically at Namenode startup, and
- | leaves safe mode automatically when the configured minimum
- | percentage of blocks satisfies the minimum replication
- | condition. Safe mode can also be entered manually, but then
- | it can only be turned off manually as well.
-*-----------------+-----------------------------------------------------------+
-| -refreshNodes | Re-read the hosts and exclude files to update the set of
- | Datanodes that are allowed to connect to the Namenode and
- | those that should be decommissioned or recommissioned.
-*-----------------+-----------------------------------------------------------+
-| -finalizeUpgrade| Finalize upgrade of HDFS. Datanodes delete their previous
- | version working directories, followed by Namenode doing the
- | same. This completes the upgrade process.
-*-----------------+-----------------------------------------------------------+
-| -upgradeProgress status / details / force | Request current distributed
- | upgrade status, a detailed status or force the upgrade to
- | proceed.
-*-----------------+-----------------------------------------------------------+
-| -metasave filename | Save Namenode's primary data structures to <filename> in
- | the directory specified by hadoop.log.dir property.
- | <filename> is overwritten if it exists.
- | <filename> will contain one line for each of the following\
- | 1. Datanodes heart beating with Namenode\
- | 2. Blocks waiting to be replicated\
- | 3. Blocks currrently being replicated\
- | 4. Blocks waiting to be deleted\
-*-----------------+-----------------------------------------------------------+
-| -setQuota <quota> <dirname>...<dirname> | Set the quota <quota> for each
- | directory <dirname>. The directory quota is a long integer
- | that puts a hard limit on the number of names in the
- | directory tree. Best effort for the directory, with faults
- | reported if \
- | 1. N is not a positive integer, or \
- | 2. user is not an administrator, or \
- | 3. the directory does not exist or is a file, or \
- | 4. the directory would immediately exceed the new quota. \
-*-----------------+-----------------------------------------------------------+
-| -clrQuota <dirname>...<dirname> | Clear the quota for each directory
- | <dirname>. Best effort for the directory. with fault
- | reported if \
- | 1. the directory does not exist or is a file, or \
- | 2. user is not an administrator. It does not fault if the
- | directory has no quota.
-*-----------------+-----------------------------------------------------------+
-| -restoreFailedStorage true / false / check | This option will turn on/off automatic attempt to restore failed storage replicas.
- | If a failed storage becomes available again the system will attempt to restore
- | edits and/or fsimage during checkpoint. 'check' option will return current setting.
-*-----------------+-----------------------------------------------------------+
-| -help [cmd] | Displays help for the given command or all commands if none
- | is specified.
-*-----------------+-----------------------------------------------------------+
-
-* <<<mradmin>>>
-
- Runs MR admin client
-
- Usage: <<<hadoop mradmin [ GENERIC_OPTIONS ] [-refreshQueueAcls]>>>
-
-*-------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-------------------+-----------------------------------------------------------+
-| -refreshQueueAcls | Refresh the queue acls used by hadoop, to check access
- | during submissions and administration of the job by the
- | user. The properties present in mapred-queue-acls.xml is
- | reloaded by the queue manager.
-*-------------------+-----------------------------------------------------------+
-
-* <<<jobtracker>>>
-
- Runs the MapReduce job Tracker node.
-
- Usage: <<<hadoop jobtracker [-dumpConfiguration]>>>
-
-*--------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*--------------------+-----------------------------------------------------------+
-| -dumpConfiguration | Dumps the configuration used by the JobTracker alongwith
- | queue configuration in JSON format into Standard output
- | used by the jobtracker and exits.
-*--------------------+-----------------------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#dfsadmin}
+ <<<hdfs dfsadmin>>>}} instead.
* <<<namenode>>>
- Runs the namenode. More info about the upgrade, rollback and finalize is
- at {{{../hadoop-hdfs/HdfsUserGuide.html#Upgrade_and_Rollback}Upgrade Rollback}}.
-
- Usage: <<<hadoop namenode [-format] | [-upgrade] | [-rollback] | [-finalize] | [-importCheckpoint]>>>
-
-*--------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*--------------------+-----------------------------------------------------------+
-| -format | Formats the namenode. It starts the namenode, formats
- | it and then shut it down.
-*--------------------+-----------------------------------------------------------+
-| -upgrade | Namenode should be started with upgrade option after
- | the distribution of new hadoop version.
-*--------------------+-----------------------------------------------------------+
-| -rollback | Rollsback the namenode to the previous version. This
- | should be used after stopping the cluster and
- | distributing the old hadoop version.
-*--------------------+-----------------------------------------------------------+
-| -finalize | Finalize will remove the previous state of the files
- | system. Recent upgrade will become permanent. Rollback
- | option will not be available anymore. After finalization
- | it shuts the namenode down.
-*--------------------+-----------------------------------------------------------+
-| -importCheckpoint | Loads image from a checkpoint directory and save it
- | into the current one. Checkpoint dir is read from
- | property fs.checkpoint.dir
-*--------------------+-----------------------------------------------------------+
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#namenode}
+ <<<hdfs namenode>>>}} instead.
* <<<secondarynamenode>>>
- Runs the HDFS secondary namenode.
- See {{{../hadoop-hdfs/HdfsUserGuide.html#Secondary_NameNode}Secondary Namenode}}
- for more info.
-
- Usage: <<<hadoop secondarynamenode [-checkpoint [force]] | [-geteditsize]>>>
-
-*----------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*----------------------+-----------------------------------------------------------+
-| -checkpoint [-force] | Checkpoints the Secondary namenode if EditLog size
- | >= fs.checkpoint.size. If <<<-force>>> is used,
- | checkpoint irrespective of EditLog size.
-*----------------------+-----------------------------------------------------------+
-| -geteditsize | Prints the EditLog size.
-*----------------------+-----------------------------------------------------------+
-
-* <<<tasktracker>>>
-
- Runs a MapReduce task Tracker node.
-
- Usage: <<<hadoop tasktracker>>>
+ Deprecated, use {{{../hadoop-hdfs/HDFSCommands.html#secondarynamenode}
+ <<<hdfs secondarynamenode>>>}} instead.
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm Tue Aug 19 23:49:39 2014
@@ -72,10 +72,13 @@ Apache Hadoop Compatibility
* Private-Stable APIs can change across major releases,
but not within a major release.
+ * Classes not annotated are implicitly "Private". Class members not
+ annotated inherit the annotations of the enclosing class.
+
* Note: APIs generated from the proto files need to be compatible for
-rolling-upgrades. See the section on wire-compatibility for more details. The
-compatibility policies for APIs and wire-communication need to go
-hand-in-hand to address this.
+ rolling-upgrades. See the section on wire-compatibility for more details.
+ The compatibility policies for APIs and wire-communication need to go
+ hand-in-hand to address this.
** Semantic compatibility
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm Tue Aug 19 23:49:39 2014
@@ -138,7 +138,7 @@ copyToLocal
count
- Usage: <<<hdfs dfs -count [-q] <paths> >>>
+ Usage: <<<hdfs dfs -count [-q] [-h] <paths> >>>
Count the number of directories, files and bytes under the paths that match
the specified file pattern. The output columns with -count are: DIR_COUNT,
@@ -147,19 +147,23 @@ count
The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
+ The -h option shows sizes in human readable format.
+
Example:
* <<<hdfs dfs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
* <<<hdfs dfs -count -q hdfs://nn1.example.com/file1>>>
+ * <<<hdfs dfs -count -q -h hdfs://nn1.example.com/file1>>>
+
Exit Code:
Returns 0 on success and -1 on error.
cp
- Usage: <<<hdfs dfs -cp [-f] URI [URI ...] <dest> >>>
+ Usage: <<<hdfs dfs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> >>>
Copy files from source to destination. This command allows multiple sources
as well in which case the destination must be a directory.
@@ -167,6 +171,12 @@ cp
Options:
* The -f option will overwrite the destination if it already exists.
+
+ * The -p option will preserve file attributes [topx] (timestamps,
+ ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
+ then preserves timestamps, ownership, permission. If -pa is specified,
+ then preserves permission also because ACL is a super-set of
+ permission.
Example:
@@ -254,6 +264,35 @@ getfacl
Returns 0 on success and non-zero on error.
+getfattr
+
+ Usage: <<<hdfs dfs -getfattr [-R] {-n name | -d} [-e en] <path> >>>
+
+ Displays the extended attribute names and values (if any) for a file or
+ directory.
+
+ Options:
+
+ * -R: Recursively list the attributes for all files and directories.
+
+ * -n name: Dump the named extended attribute value.
+
+ * -d: Dump all extended attribute values associated with pathname.
+
+ * -e <encoding>: Encode values after retrieving them. Valid encodings are "text", "hex", and "base64". Values encoded as text strings are enclosed in double quotes ("), and values encoded as hexadecimal and base64 are prefixed with 0x and 0s, respectively.
+
+ * <path>: The file or directory.
+
+ Examples:
+
+ * <<<hdfs dfs -getfattr -d /file>>>
+
+ * <<<hdfs dfs -getfattr -R -n user.myAttr /dir>>>
+
+ Exit Code:
+
+ Returns 0 on success and non-zero on error.
+
getmerge
Usage: <<<hdfs dfs -getmerge <src> <localdst> [addnl]>>>
@@ -450,6 +489,36 @@ setfacl
Returns 0 on success and non-zero on error.
+setfattr
+
+ Usage: <<<hdfs dfs -setfattr {-n name [-v value] | -x name} <path> >>>
+
+ Sets an extended attribute name and value for a file or directory.
+
+ Options:
+
+ * -b: Remove all but the base ACL entries. The entries for user, group and others are retained for compatibility with permission bits.
+
+ * -n name: The extended attribute name.
+
+ * -v value: The extended attribute value. There are three different encoding methods for the value. If the argument is enclosed in double quotes, then the value is the string inside the quotes. If the argument is prefixed with 0x or 0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 0S, then it is taken as a base64 encoding.
+
+ * -x name: Remove the extended attribute.
+
+ * <path>: The file or directory.
+
+ Examples:
+
+ * <<<hdfs dfs -setfattr -n user.myAttr -v myValue /file>>>
+
+ * <<<hdfs dfs -setfattr -n user.noValue /file>>>
+
+ * <<<hdfs dfs -setfattr -x user.myAttr /file>>>
+
+ Exit Code:
+
+ Returns 0 on success and non-zero on error.
+
setrep
Usage: <<<hdfs dfs -setrep [-R] [-w] <numReplicas> <path> >>>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm Tue Aug 19 23:49:39 2014
@@ -30,6 +30,8 @@ Native Libraries Guide
compression" could refer to all *.so's you need to compile that are
specifically related to compression. Currently, however, this document
only addresses the native hadoop library (<<<libhadoop.so>>>).
+ The document for libhdfs library (<<<libhdfs.so>>>) is
+ {{{../hadoop-hdfs/LibHdfs.html}here}}.
* Native Hadoop Library
@@ -54,24 +56,28 @@ Native Libraries Guide
[[4]] Install the compression codec development packages (>zlib-1.2,
>gzip-1.2):
- + If you download the library, install one or more development
+
+ * If you download the library, install one or more development
packages - whichever compression codecs you want to use with
your deployment.
- + If you build the library, it is mandatory to install both
+
+ * If you build the library, it is mandatory to install both
development packages.
[[5]] Check the runtime log files.
* Components
- The native hadoop library includes two components, the zlib and gzip
- compression codecs:
+ The native hadoop library includes various components:
- * zlib
+ * Compression Codecs (bzip2, lz4, snappy, zlib)
- * gzip
+ * Native IO utilities for {{{../hadoop-hdfs/ShortCircuitLocalReads.html}
+ HDFS Short-Circuit Local Reads}} and
+ {{{../hadoop-hdfs/CentralizedCacheManagement.html}Centralized Cache
+ Management in HDFS}}
- The native hadoop library is imperative for gzip to work.
+ * CRC32 checksum implementation
* Supported Platforms
@@ -116,12 +122,14 @@ Native Libraries Guide
* zlib-development package (stable version >= 1.2.0)
+ * openssl-development package(e.g. libssl-dev)
+
Once you installed the prerequisite packages use the standard hadoop
pom.xml file and pass along the native flag to build the native hadoop
library:
----
- $ mvn package -Pdist,native -Dskiptests -Dtar
+ $ mvn package -Pdist,native -DskipTests -Dtar
----
You should see the newly-built library in:
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm Tue Aug 19 23:49:39 2014
@@ -176,9 +176,11 @@ KVNO Timestamp Principal
the rule specified by <<<hadoop.security.auth_to_local>>>
which works in the same way as the <<<auth_to_local>>> in
{{{http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html}Kerberos configuration file (krb5.conf)}}.
+ In addition, Hadoop <<<auth_to_local>>> mapping supports the <</L>> flag that
+ lowercases the returned name.
By default, it picks the first component of principal name as a user name
- if the realms matches to the <<<defalut_realm>>> (usually defined in /etc/krb5.conf).
+ if the realms matches to the <<<default_realm>>> (usually defined in /etc/krb5.conf).
For example, <<<...@REALM.TLD>>> is mapped to <<<host>>>
by default rule.
@@ -201,7 +203,9 @@ KVNO Timestamp Principal
Some products such as Apache Oozie which access the services of Hadoop
on behalf of end users need to be able to impersonate end users.
You can configure proxy user using properties
- <<<hadoop.proxyuser.${superuser}.hosts>>> and <<<hadoop.proxyuser.${superuser}.groups>>>.
+ <<<hadoop.proxyuser.${superuser}.hosts>>> along with either or both of
+ <<<hadoop.proxyuser.${superuser}.groups>>>
+ and <<<hadoop.proxyuser.${superuser}.users>>>.
For example, by specifying as below in core-site.xml,
user named <<<oozie>>> accessing from any host
@@ -218,6 +222,39 @@ KVNO Timestamp Principal
</property>
----
+ User named <<<oozie>>> accessing from any host
+ can impersonate user1 and user2 by specifying as below in core-site.xml.
+
+----
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.oozie.users</name>
+ <value>user1,user2</value>
+ </property>
+----
+
+ The <<<hadoop.proxyuser.${superuser}.hosts>>> accepts list of ip addresses,
+ ip address ranges in CIDR format and/or host names.
+
+ For example, by specifying as below in core-site.xml,
+ user named <<<oozie>>> accessing from hosts in the range
+ 10.222.0.0-15 and 10.113.221.221
+ can impersonate any user belonging to any group.
+
+ ----
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
+ <value>10.222.0.0/16,10.113.221.221</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+----
+
** Secure DataNode
Because the data transfer protocol of DataNode
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm Tue Aug 19 23:49:39 2014
@@ -100,11 +100,36 @@ security.ha.service.protocol.acl |
Example: <<<user1,user2 group1,group2>>>.
Add a blank at the beginning of the line if only a list of groups is to
- be provided, equivalently a comman-separated list of users followed by
+ be provided, equivalently a comma-separated list of users followed by
a space or nothing implies only a set of given users.
A special value of <<<*>>> implies that all users are allowed to access the
- service.
+ service.
+
+ If access control list is not defined for a service, the value of
+ <<<security.service.authorization.default.acl>>> is applied. If
+ <<<security.service.authorization.default.acl>>> is not defined, <<<*>>> is applied.
+
+ ** Blocked Access Control Lists
+
+ In some cases, it is required to specify blocked access control list for a service. This specifies
+ the list of users and groups who are not authorized to access the service. The format of
+ the blocked access control list is same as that of access control list. The blocked access
+ control list can be specified via <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>>. The property name
+ is derived by suffixing with ".blocked".
+
+ Example: The property name of blocked access control list for <<<security.client.protocol.acl>>
+ will be <<<security.client.protocol.acl.blocked>>>
+
+ For a service, it is possible to specify both an access control list and a blocked control
+ list. A user is authorized to access the service if the user is in the access control and not in
+ the blocked access control list.
+
+ If blocked access control list is not defined for a service, the value of
+ <<<security.service.authorization.default.acl.blocked>>> is applied. If
+ <<<security.service.authorization.default.acl.blocked>>> is not defined,
+ empty blocked access control list is applied.
+
** Refreshing Service Level Authorization Configuration
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,210 +18,7 @@
Single Node Setup
-%{toc|section=1|fromDepth=0}
+ This page will be removed in the next major release.
-* Purpose
-
- This document describes how to set up and configure a single-node
- Hadoop installation so that you can quickly perform simple operations
- using Hadoop MapReduce and the Hadoop Distributed File System (HDFS).
-
-* Prerequisites
-
-** Supported Platforms
-
- * GNU/Linux is supported as a development and production platform.
- Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
-
- * Windows is also a supported platform.
-
-** Required Software
-
- Required software for Linux and Windows include:
-
- [[1]] Java^TM 1.6.x, preferably from Sun, must be installed.
-
- [[2]] ssh must be installed and sshd must be running to use the Hadoop
- scripts that manage remote Hadoop daemons.
-
-** Installing Software
-
- If your cluster doesn't have the requisite software you will need to
- install it.
-
- For example on Ubuntu Linux:
-
-----
- $ sudo apt-get install ssh
- $ sudo apt-get install rsync
-----
-
-* Download
-
- To get a Hadoop distribution, download a recent stable release from one
- of the Apache Download Mirrors.
-
-* Prepare to Start the Hadoop Cluster
-
- Unpack the downloaded Hadoop distribution. In the distribution, edit
- the file <<<conf/hadoop-env.sh>>> to define at least <<<JAVA_HOME>>> to be the root
- of your Java installation.
-
- Try the following command:
-
-----
- $ bin/hadoop
-----
-
- This will display the usage documentation for the hadoop script.
-
- Now you are ready to start your Hadoop cluster in one of the three
- supported modes:
-
- * Local (Standalone) Mode
-
- * Pseudo-Distributed Mode
-
- * Fully-Distributed Mode
-
-* Standalone Operation
-
- By default, Hadoop is configured to run in a non-distributed mode, as a
- single Java process. This is useful for debugging.
-
- The following example copies the unpacked conf directory to use as
- input and then finds and displays every match of the given regular
- expression. Output is written to the given output directory.
-
-----
- $ mkdir input
- $ cp conf/*.xml input
- $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+'
- $ cat output/*
----
-
-* Pseudo-Distributed Operation
-
- Hadoop can also be run on a single-node in a pseudo-distributed mode
- where each Hadoop daemon runs in a separate Java process.
-
-** Configuration
-
- Use the following:
-
- conf/core-site.xml:
-
-----
-<configuration>
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://localhost:9000</value>
- </property>
-</configuration>
-----
-
- conf/hdfs-site.xml:
-
-----
-<configuration>
- <property>
- <name>dfs.replication</name>
- <value>1</value>
- </property>
-</configuration>
-----
-
- conf/mapred-site.xml:
-
-----
-<configuration>
- <property>
- <name>mapred.job.tracker</name>
- <value>localhost:9001</value>
- </property>
-</configuration>
-----
-
-** Setup passphraseless ssh
-
- Now check that you can ssh to the localhost without a passphrase:
-
-----
- $ ssh localhost
-----
-
- If you cannot ssh to localhost without a passphrase, execute the
- following commands:
-
-----
- $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
- $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
-----
-
-** Execution
-
- Format a new distributed-filesystem:
-
-----
- $ bin/hadoop namenode -format
-----
-
- Start the hadoop daemons:
-
-----
- $ bin/start-all.sh
-----
-
- The hadoop daemon log output is written to the <<<${HADOOP_LOG_DIR}>>>
- directory (defaults to <<<${HADOOP_PREFIX}/logs>>>).
-
- Browse the web interface for the NameNode and the JobTracker; by
- default they are available at:
-
- * NameNode - <<<http://localhost:50070/>>>
-
- * JobTracker - <<<http://localhost:50030/>>>
-
- Copy the input files into the distributed filesystem:
-
-----
- $ bin/hadoop fs -put conf input
-----
-
- Run some of the examples provided:
-
-----
- $ bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+'
-----
-
- Examine the output files:
-
- Copy the output files from the distributed filesystem to the local
- filesytem and examine them:
-
-----
- $ bin/hadoop fs -get output output
- $ cat output/*
-----
-
- or
-
- View the output files on the distributed filesystem:
-
-----
- $ bin/hadoop fs -cat output/*
-----
-
- When you're done, stop the daemons with:
-
-----
- $ bin/stop-all.sh
-----
-
-* Fully-Distributed Operation
-
- For information on setting up fully-distributed, non-trivial clusters
- see {{{./ClusterSetup.html}Cluster Setup}}.
-
- Java and JNI are trademarks or registered trademarks of Sun
- Microsystems, Inc. in the United States and other countries.
+ See {{{./SingleCluster.html}Single Cluster Setup}} to set up and configure a
+ single-node Hadoop installation.
Propchange: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-2006/hadoop-common-project/hadoop-common/src/test/core:r1588992-1596568
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1582150-1619000
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java Tue Aug 19 23:49:39 2014
@@ -49,7 +49,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
-import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
@@ -178,6 +178,14 @@ public class TestConfiguration extends T
// check that expansion also occurs for getInt()
assertTrue(conf.getInt("intvar", -1) == 42);
assertTrue(conf.getInt("my.int", -1) == 42);
+
+ Map<String, String> results = conf.getValByRegex("^my.*file$");
+ assertTrue(results.keySet().contains("my.relfile"));
+ assertTrue(results.keySet().contains("my.fullfile"));
+ assertTrue(results.keySet().contains("my.file"));
+ assertEquals(-1, results.get("my.relfile").indexOf("${"));
+ assertEquals(-1, results.get("my.fullfile").indexOf("${"));
+ assertEquals(-1, results.get("my.file").indexOf("${"));
}
public void testFinalParam() throws IOException {
@@ -1003,6 +1011,14 @@ public class TestConfiguration extends T
String resource;
}
+ public void testGetSetTrimmedNames() throws IOException {
+ Configuration conf = new Configuration(false);
+ conf.set(" name", "value");
+ assertEquals("value", conf.get("name"));
+ assertEquals("value", conf.get(" name"));
+ assertEquals("value", conf.getRaw(" name "));
+ }
+
public void testDumpConfiguration () throws IOException {
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java Tue Aug 19 23:49:39 2014
@@ -38,7 +38,7 @@ import java.util.concurrent.ScheduledThr
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration.DeprecationDelta;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java Tue Aug 19 23:49:39 2014
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.crypto.key;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.junit.Test;
import java.io.IOException;
@@ -29,7 +30,9 @@ import java.security.NoSuchAlgorithmExce
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@@ -61,7 +64,7 @@ public class TestKeyProvider {
@Test
public void testKeyMaterial() throws Exception {
byte[] key1 = new byte[]{1,2,3,4};
- KeyProvider.KeyVersion obj = new KeyProvider.KeyVersion("key1@1", key1);
+ KeyProvider.KeyVersion obj = new KeyProvider.KeyVersion("key1", "key1@1", key1);
assertEquals("key1@1", obj.getVersionName());
assertArrayEquals(new byte[]{1,2,3,4}, obj.getMaterial());
}
@@ -72,7 +75,7 @@ public class TestKeyProvider {
DateFormat format = new SimpleDateFormat("y/m/d");
Date date = format.parse("2013/12/25");
KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null,
- date, 123);
+ null, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertNull(meta.getDescription());
@@ -82,6 +85,7 @@ public class TestKeyProvider {
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertNull(second.getDescription());
+ assertTrue(second.getAttributes().isEmpty());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
int newVersion = second.addVersion();
@@ -92,17 +96,21 @@ public class TestKeyProvider {
//Metadata with description
format = new SimpleDateFormat("y/m/d");
date = format.parse("2013/12/25");
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("a", "A");
meta = new KeyProvider.Metadata("myCipher", 100,
- "description", date, 123);
+ "description", attributes, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertEquals("description", meta.getDescription());
+ assertEquals(attributes, meta.getAttributes());
assertEquals(date, meta.getCreated());
assertEquals(123, meta.getVersions());
second = new KeyProvider.Metadata(meta.serialize());
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertEquals(meta.getDescription(), second.getDescription());
+ assertEquals(meta.getAttributes(), second.getAttributes());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
newVersion = second.addVersion();
@@ -116,15 +124,19 @@ public class TestKeyProvider {
Configuration conf = new Configuration();
conf.set(KeyProvider.DEFAULT_CIPHER_NAME, "myCipher");
conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 512);
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("a", "A");
KeyProvider.Options options = KeyProvider.options(conf);
assertEquals("myCipher", options.getCipher());
assertEquals(512, options.getBitLength());
options.setCipher("yourCipher");
options.setDescription("description");
+ options.setAttributes(attributes);
options.setBitLength(128);
assertEquals("yourCipher", options.getCipher());
assertEquals(128, options.getBitLength());
assertEquals("description", options.getDescription());
+ assertEquals(attributes, options.getAttributes());
options = KeyProvider.options(new Configuration());
assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher());
assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength());
@@ -133,13 +145,13 @@ public class TestKeyProvider {
@Test
public void testUnnestUri() throws Exception {
assertEquals(new Path("hdfs://nn.example.com/my/path"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
assertEquals(new Path("hdfs://nn/my/path?foo=bar&baz=bat#yyy"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
assertEquals(new Path("inner://hdfs@nn1.example.com/my/path"),
- KeyProvider.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
assertEquals(new Path("user:///"),
- KeyProvider.unnestUri(new URI("outer://user/")));
+ ProviderUtils.unnestUri(new URI("outer://user/")));
}
private static class MyKeyProvider extends KeyProvider {
@@ -166,7 +178,7 @@ public class TestKeyProvider {
@Override
public Metadata getMetadata(String name) throws IOException {
- return new Metadata(CIPHER, 128, "description", new Date(), 0);
+ return new Metadata(CIPHER, 128, "description", null, new Date(), 0);
}
@Override
Modified: hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java Tue Aug 19 23:49:39 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Before;
@@ -99,9 +100,9 @@ public class TestKeyProviderFactory {
static void checkSpecificProvider(Configuration conf,
String ourUrl) throws Exception {
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
- byte[] key1 = new byte[32];
- byte[] key2 = new byte[32];
- byte[] key3 = new byte[32];
+ byte[] key1 = new byte[16];
+ byte[] key2 = new byte[16];
+ byte[] key3 = new byte[16];
for(int i =0; i < key1.length; ++i) {
key1[i] = (byte) i;
key2[i] = (byte) (i * 2);
@@ -145,7 +146,7 @@ public class TestKeyProviderFactory {
KeyProvider.options(conf).setBitLength(8));
assertTrue("should throw", false);
} catch (IOException e) {
- assertEquals("Wrong key length. Required 8, but got 256", e.getMessage());
+ assertEquals("Wrong key length. Required 8, but got 128", e.getMessage());
}
provider.createKey("key4", new byte[]{1},
KeyProvider.options(conf).setBitLength(8));
@@ -161,7 +162,7 @@ public class TestKeyProviderFactory {
provider.rollNewVersion("key4", key1);
assertTrue("should throw", false);
} catch (IOException e) {
- assertEquals("Wrong key length. Required 8, but got 256", e.getMessage());
+ assertEquals("Wrong key length. Required 8, but got 128", e.getMessage());
}
try {
provider.rollNewVersion("no-such-key", key1);
@@ -213,21 +214,86 @@ public class TestKeyProviderFactory {
file.delete();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
checkSpecificProvider(conf, ourUrl);
- Path path = KeyProvider.unnestUri(new URI(ourUrl));
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------"));
assertTrue(file + " should exist", file.isFile());
+ // Corrupt file and Check if JKS can reload from _OLD file
+ File oldFile = new File(file.getPath() + "_OLD");
+ file.renameTo(oldFile);
+ file.delete();
+ file.createNewFile();
+ assertTrue(oldFile.exists());
+ KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+ assertTrue(file.exists());
+ assertTrue(oldFile + "should be deleted", !oldFile.exists());
+ verifyAfterReload(file, provider);
+ assertTrue(!oldFile.exists());
+
+ // _NEW and current file should not exist together
+ File newFile = new File(file.getPath() + "_NEW");
+ newFile.createNewFile();
+ try {
+ provider = KeyProviderFactory.getProviders(conf).get(0);
+ Assert.fail("_NEW and current file should not exist together !!");
+ } catch (Exception e) {
+ // Ignore
+ } finally {
+ if (newFile.exists()) {
+ newFile.delete();
+ }
+ }
+
+ // Load from _NEW file
+ file.renameTo(newFile);
+ file.delete();
+ try {
+ provider = KeyProviderFactory.getProviders(conf).get(0);
+ Assert.assertFalse(newFile.exists());
+ Assert.assertFalse(oldFile.exists());
+ } catch (Exception e) {
+ Assert.fail("JKS should load from _NEW file !!");
+ // Ignore
+ }
+ verifyAfterReload(file, provider);
+
+ // _NEW exists but corrupt.. must load from _OLD
+ newFile.createNewFile();
+ file.renameTo(oldFile);
+ file.delete();
+ try {
+ provider = KeyProviderFactory.getProviders(conf).get(0);
+ Assert.assertFalse(newFile.exists());
+ Assert.assertFalse(oldFile.exists());
+ } catch (Exception e) {
+ Assert.fail("JKS should load from _OLD file !!");
+ // Ignore
+ } finally {
+ if (newFile.exists()) {
+ newFile.delete();
+ }
+ }
+ verifyAfterReload(file, provider);
+
// check permission retention after explicit change
fs.setPermission(path, new FsPermission("777"));
checkPermissionRetention(conf, ourUrl, path);
}
+ private void verifyAfterReload(File file, KeyProvider provider)
+ throws IOException {
+ List<String> existingKeys = provider.getKeys();
+ assertTrue(existingKeys.contains("key4"));
+ assertTrue(existingKeys.contains("key3"));
+ assertTrue(file.exists());
+ }
+
public void checkPermissionRetention(Configuration conf, String ourUrl, Path path) throws Exception {
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
// let's add a new key and flush and check that permissions are still set to 777
- byte[] key = new byte[32];
+ byte[] key = new byte[16];
for(int i =0; i < key.length; ++i) {
key[i] = (byte) i;
}
@@ -260,7 +326,7 @@ public class TestKeyProviderFactory {
conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY,
"javakeystoreprovider.password");
KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
- provider.createKey("key3", new byte[32], KeyProvider.options(conf));
+ provider.createKey("key3", new byte[16], KeyProvider.options(conf));
provider.flush();
} catch (Exception ex) {
Assert.fail("could not create keystore with password file");