You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2014/10/10 21:00:40 UTC
[01/50] [abbrv] git commit: HDFS-6664. HDFS permissions guide
documentation states incorrect default group mapping class. (Ray Chiang via
aw)
Repository: hadoop
Updated Branches:
refs/heads/champlain [created] 51c8d1737
HDFS-6664. HDFS permissions guide documentation states incorrect default group mapping class. (Ray Chiang via aw)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a805d524
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a805d524
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a805d524
Branch: refs/heads/champlain
Commit: a805d5241407a10164eebde4f90fd1a1d04a2d04
Parents: 395605b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 26 12:21:37 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:46:00 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/site/apt/HdfsPermissionsGuide.apt.vm | 13 ++++++++++---
2 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a805d524/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f53b5b2..0a37e7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -755,6 +755,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6904. YARN unable to renew delegation token fetched via webhdfs due to
incorrect service port (jitendra)
+ HDFS-6664. HDFS permissions guide documentation states incorrect default
+ group mapping class. (Ray Chiang via aw)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a805d524/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsPermissionsGuide.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsPermissionsGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsPermissionsGuide.apt.vm
index f9aac0b..30119a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsPermissionsGuide.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsPermissionsGuide.apt.vm
@@ -99,9 +99,16 @@ HDFS Permissions Guide
Once a username has been determined as described above, the list of
groups is determined by a group mapping service, configured by the
hadoop.security.group.mapping property. The default implementation,
- org.apache.hadoop.security.ShellBasedUnixGroupsMapping, will shell out
- to the Unix bash -c groups command to resolve a list of groups for a
- user.
+ org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
+ will determine if the Java Native Interface (JNI) is available. If
+ JNI is available, the implementation will use the API within hadoop
+ to resolve a list of groups for a user. If JNI is not available
+ then the shell implementation,
+ org.apache.hadoop.security.ShellBasedUnixGroupsMapping, is used.
+ This implementation shells out with the <<<bash -c groups>>>
+ command (for a Linux/Unix environment) or the <<<net group>>>
+ command (for a Windows environment) to resolve a list of groups for
+ a user.
An alternate implementation, which connects directly to an LDAP server
to resolve the list of groups, is available via
[37/50] [abbrv] git commit: HDFS-7195. Update user doc of secure mode
about Datanodes don't require root or jsvc. Contributed by Chris Nauroth.
Posted by st...@apache.org.
HDFS-7195. Update user doc of secure mode about Datanodes don't require root or jsvc. Contributed by Chris Nauroth.
(cherry picked from commit 4e544c84e9e2cf3fb031cd8797bac4e8bcc7eca9)
(cherry picked from commit 754718aed56480135acd0f32878f1fe4607c34df)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45eff1ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45eff1ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45eff1ad
Branch: refs/heads/champlain
Commit: 45eff1ade7997de4e34b6f23c249f3535797c250
Parents: 5a5e0d4
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 11:55:59 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 11:59:58 2014 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/hadoop-env.sh | 11 ++++-
.../src/site/apt/SecureMode.apt.vm | 47 +++++++++++++++++++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
3 files changed, 58 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45eff1ad/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index e964ec0..f60b65a 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -24,7 +24,10 @@
# The java implementation to use.
export JAVA_HOME=${JAVA_HOME}
-# The jsvc implementation to use. Jsvc is required to run secure datanodes.
+# The jsvc implementation to use. Jsvc is required to run secure datanodes
+# that bind to privileged ports to provide authentication of data transfer
+# protocol. Jsvc is not required if SASL is configured for authentication of
+# data transfer protocol using non-privileged ports.
#export JSVC_HOME=${JSVC_HOME}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
@@ -58,7 +61,11 @@ export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
-# On secure datanodes, user to run the datanode as after dropping privileges
+# On secure datanodes, user to run the datanode as after dropping privileges.
+# This **MUST** be uncommented to enable secure HDFS if using privileged ports
+# to provide authentication of data transfer protocol. This **MUST NOT** be
+# defined if SASL is configured for authentication of data transfer protocol
+# using non-privileged ports.
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
# Where log files are stored. $HADOOP_HOME/logs by default.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45eff1ad/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
index 54473e6..2085d5b 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
@@ -244,7 +244,7 @@ KVNO Timestamp Principal
10.222.0.0-15 and 10.113.221.221
can impersonate any user belonging to any group.
- ----
+----
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
<value>10.222.0.0/16,10.113.221.221</value>
@@ -273,6 +273,33 @@ KVNO Timestamp Principal
You must specify <<<HADOOP_SECURE_DN_USER>>> and <<<JSVC_HOME>>>
as environment variables on start up (in hadoop-env.sh).
+ As of version 2.6.0, SASL can be used to authenticate the data transfer
+ protocol. In this configuration, it is no longer required for secured clusters
+ to start the DataNode as root using jsvc and bind to privileged ports. To
+ enable SASL on data transfer protocol, set <<<dfs.data.transfer.protection>>>
+ in hdfs-site.xml, set a non-privileged port for <<<dfs.datanode.address>>>, set
+ <<<dfs.http.policy>>> to <HTTPS_ONLY> and make sure the
+ <<<HADOOP_SECURE_DN_USER>>> environment variable is not defined. Note that it
+ is not possible to use SASL on data transfer protocol if
+ <<<dfs.datanode.address>>> is set to a privileged port. This is required for
+ backwards-compatibility reasons.
+
+ In order to migrate an existing cluster that used root authentication to start
+ using SASL instead, first ensure that version 2.6.0 or later has been deployed
+ to all cluster nodes as well as any external applications that need to connect
+ to the cluster. Only versions 2.6.0 and later of the HDFS client can connect
+ to a DataNode that uses SASL for authentication of data transfer protocol, so
+ it is vital that all callers have the correct version before migrating. After
+ version 2.6.0 or later has been deployed everywhere, update configuration of
+ any external applications to enable SASL. If an HDFS client is enabled for
+ SASL, then it can connect successfully to a DataNode running with either root
+ authentication or SASL authentication. Changing configuration for all clients
+ guarantees that subsequent configuration changes on DataNodes will not disrupt
+ the applications. Finally, each individual DataNode can be migrated by
+ changing its configuration and restarting. It is acceptable to have a mix of
+ some DataNodes running with root authentication and some DataNodes running with
+ SASL authentication temporarily during this migration period, because an HDFS
+ client enabled for SASL can connect to both.
* Data confidentiality
@@ -391,6 +418,10 @@ Configuration for <<<conf/core-site.xml>>>
| <<<dfs.http.policy>>> | <HTTP_ONLY> or <HTTPS_ONLY> or <HTTP_AND_HTTPS> | |
| | | HTTPS_ONLY turns off http access. This option takes precedence over |
| | | the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. |
+| | | If using SASL to authenticate data transfer protocol instead of |
+| | | running DataNode as root and using privileged ports, then this property |
+| | | must be set to <HTTPS_ONLY> to guarantee authentication of HTTP servers. |
+| | | (See <<<dfs.data.transfer.protection>>>.) |
*-------------------------+-------------------------+------------------------+
| <<<dfs.namenode.https-address>>> | <nn_host_fqdn:50470> | |
*-------------------------+-------------------------+------------------------+
@@ -440,6 +471,9 @@ Configuration for <<<conf/hdfs-site.xml>>>
| | | Secure DataNode must use privileged port |
| | | in order to assure that the server was started securely. |
| | | This means that the server must be started via jsvc. |
+| | | Alternatively, this must be set to a non-privileged port if using SASL |
+| | | to authenticate data transfer protocol. |
+| | | (See <<<dfs.data.transfer.protection>>>.) |
*-------------------------+-------------------------+------------------------+
| <<<dfs.datanode.http.address>>> | <0.0.0.0:1006> | |
| | | Secure DataNode must use privileged port |
@@ -457,6 +491,17 @@ Configuration for <<<conf/hdfs-site.xml>>>
| <<<dfs.encrypt.data.transfer>>> | <false> | |
| | | set to <<<true>>> when using data encryption |
*-------------------------+-------------------------+------------------------+
+| <<<dfs.data.transfer.protection>>> | | |
+| | | <authentication> : authentication only \
+| | | <integrity> : integrity check in addition to authentication \
+| | | <privacy> : data encryption in addition to integrity |
+| | | This property is unspecified by default. Setting this property enables |
+| | | SASL for authentication of data transfer protocol. If this is enabled, |
+| | | then <<<dfs.datanode.address>>> must use a non-privileged port, |
+| | | <<<dfs.http.policy>>> must be set to <HTTPS_ONLY> and the |
+| | | <<<HADOOP_SECURE_DN_USER>>> environment variable must be undefined when |
+| | | starting the DataNode process. |
+*-------------------------+-------------------------+------------------------+
Configuration for <<<conf/hdfs-site.xml>>>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45eff1ad/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a30f3b9..bc16770 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -294,6 +294,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6779. Add missing version subcommand for hdfs.
(Sasaki Toru via wheat9)
+ HDFS-7195. Update user doc of secure mode about Datanodes don't require root
+ or jsvc. (cnauroth)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
[50/50] [abbrv] git commit: YARN-2668-002.patch: rm assert in
TestSecureRMRegistryOperations
Posted by st...@apache.org.
YARN-2668-002.patch: rm assert in TestSecureRMRegistryOperations
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51c8d173
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51c8d173
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51c8d173
Branch: refs/heads/champlain
Commit: 51c8d17373c92d0912ed85b1acee8bc5cf364862
Parents: b774d29
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 10 11:58:53 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 10 11:59:54 2014 -0700
----------------------------------------------------------------------
.../hadoop/registry/secure/TestSecureRMRegistryOperations.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51c8d173/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
index bb95340..41760d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java
@@ -138,8 +138,6 @@ public class TestSecureRMRegistryOperations extends AbstractSecureRegistryTest {
assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
RegistrySecurity.isClientSASLEnabled());
- assertFalse("ZooKeeperSaslClient.isEnabled()==true",
- ZooKeeperSaslClient.isEnabled());
operations.list(PATH_SYSTEM_SERVICES);
}
[38/50] [abbrv] git commit: BUG-24485 Change Windows MSI
installations to set LzoCodec settings in core-site.xml.
Posted by st...@apache.org.
BUG-24485 Change Windows MSI installations to set LzoCodec settings in core-site.xml.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b08182c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b08182c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b08182c5
Branch: refs/heads/champlain
Commit: b08182c524247ab069fc89925acbef9eae10bb1e
Parents: 45eff1a
Author: Ashish Singh <as...@hortonworks.com>
Authored: Thu Oct 9 15:37:30 2014 -0700
Committer: Ashish Singh <as...@hortonworks.com>
Committed: Thu Oct 9 15:37:30 2014 -0700
----------------------------------------------------------------------
hadoop-dist/src/win/scripts/install.ps1 | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b08182c5/hadoop-dist/src/win/scripts/install.ps1
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/win/scripts/install.ps1 b/hadoop-dist/src/win/scripts/install.ps1
index 4763036..d9c54e0 100644
--- a/hadoop-dist/src/win/scripts/install.ps1
+++ b/hadoop-dist/src/win/scripts/install.ps1
@@ -209,7 +209,7 @@ function Main( $scriptDir )
$coreConfigs["io.compression.codec.lzo.class"] = "com.hadoop.compression.lzo.LzoCodec"
}
if ((Test-Path ENV:ENABLE_GZIP) -and ($ENV:ENABLE_GZIP -ieq "yes")){
- $coreConfigs["io.compression.codecs"] = "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec"
+ $coreConfigs["io.compression.codecs"] = "org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec"
}
if ($ENV:DEFAULT_FS -eq "ASV"){
Write-Log "ASV usage detected. Configuring HDP to use ASV as default filesystem"
[11/50] [abbrv] git commit: HDFS-7157. Using Time.now() for recording
start/end time of reconfiguration tasks (Lei Xu via Colin Patrick McCabe)
(cherry picked from commit b3d5d269a73a1826ebf6f637a0dbd69f6261dbcb)
Posted by st...@apache.org.
HDFS-7157. Using Time.now() for recording start/end time of reconfiguration tasks (Lei Xu via Colin Patrick McCabe)
(cherry picked from commit b3d5d269a73a1826ebf6f637a0dbd69f6261dbcb)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5882ddf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5882ddf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5882ddf
Branch: refs/heads/champlain
Commit: b5882ddf6b67c39e2fd7064b27a94f398773f9e2
Parents: 3373958
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Sep 29 11:29:54 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:07:46 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5882ddf/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 68a145d..f3ef5ab 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -435,6 +435,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11160. Fix typo in nfs3 server duplicate entry reporting.
(Charles Lamb via wheat9)
+ HDFS-7157. Using Time.now() for recording start/end time of reconfiguration
+ tasks (Lei Xu via cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5882ddf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
index 7521650..3248b9a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
@@ -134,7 +134,7 @@ public abstract class ReconfigurableBase
}
synchronized (this.parent.reconfigLock) {
- this.parent.endTime = Time.monotonicNow();
+ this.parent.endTime = Time.now();
this.parent.status = Collections.unmodifiableMap(results);
this.parent.reconfigThread = null;
}
@@ -160,7 +160,7 @@ public abstract class ReconfigurableBase
reconfigThread.setDaemon(true);
reconfigThread.setName("Reconfiguration Task");
reconfigThread.start();
- startTime = Time.monotonicNow();
+ startTime = Time.now();
}
}
[27/50] [abbrv] git commit: HDFS-7176. The namenode usage message
doesn't include "-rollingupgrade started" (cmccabe) (cherry picked from
commit dd1b8f2ed8a86871517c730a9f370aca4b763514) (cherry picked from commit
1725d56132c54d9ccab9e25b1ea5ad5c04c43795
Posted by st...@apache.org.
HDFS-7176. The namenode usage message doesn't include "-rollingupgrade started" (cmccabe)
(cherry picked from commit dd1b8f2ed8a86871517c730a9f370aca4b763514)
(cherry picked from commit 1725d56132c54d9ccab9e25b1ea5ad5c04c43795)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa9e4e2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa9e4e2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa9e4e2c
Branch: refs/heads/champlain
Commit: fa9e4e2cdbf04314faa5ac740be3b238d3d29f21
Parents: 54465f7
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Oct 1 14:53:44 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:59:26 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/namenode/NameNode.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa9e4e2c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7a33eeb..723aafc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -550,6 +550,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7162. Wrong path when deleting through fuse-dfs a file which already
exists in trash (Chenging Liu via cmccabe)
+ HDFS-7176. The namenode usage message doesn't include "-rollingupgrade
+ started" (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa9e4e2c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 523529d..ac86e6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -217,7 +217,8 @@ public class NameNode implements NameNodeStatusMXBean {
+ StartupOption.ROLLBACK.getName() + "] | \n\t["
+ StartupOption.ROLLINGUPGRADE.getName() + " <"
+ RollingUpgradeStartupOption.DOWNGRADE.name().toLowerCase() + "|"
- + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | \n\t["
+ + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "|"
+ + RollingUpgradeStartupOption.STARTED.name().toLowerCase() + "> ] | \n\t["
+ StartupOption.FINALIZE.getName() + "] | \n\t["
+ StartupOption.IMPORT.getName() + "] | \n\t["
+ StartupOption.INITIALIZESHAREDEDITS.getName() + "] | \n\t["
[29/50] [abbrv] git commit: HADOOP-10681. Remove unnecessary
synchronization from Snappy & Zlib codecs. Contributed by Gopal
Vijayaraghavan.
Posted by st...@apache.org.
HADOOP-10681. Remove unnecessary synchronization from Snappy & Zlib codecs. Contributed by Gopal Vijayaraghavan.
(cherry picked from commit 8f9ab998e273259c1e7a3ed53ba37d767e02b6bb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23cd97aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23cd97aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23cd97aa
Branch: refs/heads/champlain
Commit: 23cd97aabc2107a5c3ac10636b0671aa5e5e6026
Parents: 4baf131
Author: Arun C. Murthy <ac...@apache.org>
Authored: Sun Oct 5 07:38:21 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:10:23 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 ++
.../io/compress/snappy/SnappyCompressor.java | 24 +++---
.../io/compress/snappy/SnappyDecompressor.java | 26 +++---
.../hadoop/io/compress/zlib/ZlibCompressor.java | 24 +++---
.../io/compress/zlib/ZlibDecompressor.java | 30 +++----
.../hadoop/io/compress/TestCodecPool.java | 87 ++++++++++++++++++++
6 files changed, 145 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e7e972ad..04cc594 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -290,6 +290,12 @@ Release 2.6.0 - UNRELEASED
HADOOP-11111 MiniKDC to use locale EN_US for case conversions. (stevel)
+ HADOOP-10731. Remove @date JavaDoc comment in ProgramDriver class (Henry
+ Saputra via aw)
+
+ HADOOP-10681. Remove unnecessary synchronization from Snappy & Zlib
+ codecs. (Gopal Vijayaraghavan via acmurthy)
+
BUG FIXES
HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
index 376ea06..ab45f25 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
@@ -100,7 +100,7 @@ public class SnappyCompressor implements Compressor {
* @param len Length
*/
@Override
- public synchronized void setInput(byte[] b, int off, int len) {
+ public void setInput(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
@@ -127,7 +127,7 @@ public class SnappyCompressor implements Compressor {
* aside to be loaded by this function while the compressed data are
* consumed.
*/
- synchronized void setInputFromSavedData() {
+ void setInputFromSavedData() {
if (0 >= userBufLen) {
return;
}
@@ -146,7 +146,7 @@ public class SnappyCompressor implements Compressor {
* Does nothing.
*/
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
// do nothing
}
@@ -158,7 +158,7 @@ public class SnappyCompressor implements Compressor {
* #setInput() should be called in order to provide more input.
*/
@Override
- public synchronized boolean needsInput() {
+ public boolean needsInput() {
return !(compressedDirectBuf.remaining() > 0
|| uncompressedDirectBuf.remaining() == 0 || userBufLen > 0);
}
@@ -168,7 +168,7 @@ public class SnappyCompressor implements Compressor {
* with the current contents of the input buffer.
*/
@Override
- public synchronized void finish() {
+ public void finish() {
finish = true;
}
@@ -180,7 +180,7 @@ public class SnappyCompressor implements Compressor {
* data output stream has been reached.
*/
@Override
- public synchronized boolean finished() {
+ public boolean finished() {
// Check if all uncompressed data has been consumed
return (finish && finished && compressedDirectBuf.remaining() == 0);
}
@@ -197,7 +197,7 @@ public class SnappyCompressor implements Compressor {
* @return The actual number of bytes of compressed data.
*/
@Override
- public synchronized int compress(byte[] b, int off, int len)
+ public int compress(byte[] b, int off, int len)
throws IOException {
if (b == null) {
throw new NullPointerException();
@@ -250,7 +250,7 @@ public class SnappyCompressor implements Compressor {
* Resets compressor so that a new set of input data can be processed.
*/
@Override
- public synchronized void reset() {
+ public void reset() {
finish = false;
finished = false;
uncompressedDirectBuf.clear();
@@ -268,7 +268,7 @@ public class SnappyCompressor implements Compressor {
* @param conf Configuration from which new setting are fetched
*/
@Override
- public synchronized void reinit(Configuration conf) {
+ public void reinit(Configuration conf) {
reset();
}
@@ -276,7 +276,7 @@ public class SnappyCompressor implements Compressor {
* Return number of bytes given to this compressor since last reset.
*/
@Override
- public synchronized long getBytesRead() {
+ public long getBytesRead() {
return bytesRead;
}
@@ -284,7 +284,7 @@ public class SnappyCompressor implements Compressor {
* Return number of bytes consumed by callers of compress since last reset.
*/
@Override
- public synchronized long getBytesWritten() {
+ public long getBytesWritten() {
return bytesWritten;
}
@@ -292,7 +292,7 @@ public class SnappyCompressor implements Compressor {
* Closes the compressor and discards any unprocessed input.
*/
@Override
- public synchronized void end() {
+ public void end() {
}
private native static void initIDs();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
index a867717..b5f5acf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
@@ -103,7 +103,7 @@ public class SnappyDecompressor implements Decompressor {
* @param len Length
*/
@Override
- public synchronized void setInput(byte[] b, int off, int len) {
+ public void setInput(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
@@ -127,7 +127,7 @@ public class SnappyDecompressor implements Decompressor {
* aside to be loaded by this function while the compressed data are
* consumed.
*/
- synchronized void setInputFromSavedData() {
+ void setInputFromSavedData() {
compressedDirectBufLen = Math.min(userBufLen, directBufferSize);
// Reinitialize snappy's input direct buffer
@@ -144,7 +144,7 @@ public class SnappyDecompressor implements Decompressor {
* Does nothing.
*/
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
// do nothing
}
@@ -158,7 +158,7 @@ public class SnappyDecompressor implements Decompressor {
* order to provide more input.
*/
@Override
- public synchronized boolean needsInput() {
+ public boolean needsInput() {
// Consume remaining compressed data?
if (uncompressedDirectBuf.remaining() > 0) {
return false;
@@ -183,7 +183,7 @@ public class SnappyDecompressor implements Decompressor {
* @return <code>false</code>.
*/
@Override
- public synchronized boolean needsDictionary() {
+ public boolean needsDictionary() {
return false;
}
@@ -195,7 +195,7 @@ public class SnappyDecompressor implements Decompressor {
* data output stream has been reached.
*/
@Override
- public synchronized boolean finished() {
+ public boolean finished() {
return (finished && uncompressedDirectBuf.remaining() == 0);
}
@@ -212,7 +212,7 @@ public class SnappyDecompressor implements Decompressor {
* @throws IOException
*/
@Override
- public synchronized int decompress(byte[] b, int off, int len)
+ public int decompress(byte[] b, int off, int len)
throws IOException {
if (b == null) {
throw new NullPointerException();
@@ -257,13 +257,13 @@ public class SnappyDecompressor implements Decompressor {
* @return <code>0</code>.
*/
@Override
- public synchronized int getRemaining() {
+ public int getRemaining() {
// Never use this function in BlockDecompressorStream.
return 0;
}
@Override
- public synchronized void reset() {
+ public void reset() {
finished = false;
compressedDirectBufLen = 0;
uncompressedDirectBuf.limit(directBufferSize);
@@ -276,7 +276,7 @@ public class SnappyDecompressor implements Decompressor {
* input data can be processed.
*/
@Override
- public synchronized void end() {
+ public void end() {
// do nothing
}
@@ -333,7 +333,7 @@ public class SnappyDecompressor implements Decompressor {
private boolean endOfInput;
@Override
- public synchronized void decompress(ByteBuffer src, ByteBuffer dst)
+ public void decompress(ByteBuffer src, ByteBuffer dst)
throws IOException {
assert dst.isDirect() : "dst.isDirect()";
assert src.isDirect() : "src.isDirect()";
@@ -343,13 +343,13 @@ public class SnappyDecompressor implements Decompressor {
}
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
throw new UnsupportedOperationException(
"byte[] arrays are not supported for DirectDecompressor");
}
@Override
- public synchronized int decompress(byte[] b, int off, int len) {
+ public int decompress(byte[] b, int off, int len) {
throw new UnsupportedOperationException(
"byte[] arrays are not supported for DirectDecompressor");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
index ce7f68d..6799403 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
@@ -243,7 +243,7 @@ public class ZlibCompressor implements Compressor {
* @param conf Configuration storing new settings
*/
@Override
- public synchronized void reinit(Configuration conf) {
+ public void reinit(Configuration conf) {
reset();
if (conf == null) {
return;
@@ -260,7 +260,7 @@ public class ZlibCompressor implements Compressor {
}
@Override
- public synchronized void setInput(byte[] b, int off, int len) {
+ public void setInput(byte[] b, int off, int len) {
if (b== null) {
throw new NullPointerException();
}
@@ -280,7 +280,7 @@ public class ZlibCompressor implements Compressor {
}
//copy enough data from userBuf to uncompressedDirectBuf
- synchronized void setInputFromSavedData() {
+ void setInputFromSavedData() {
int len = Math.min(userBufLen, uncompressedDirectBuf.remaining());
((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff, len);
userBufLen -= len;
@@ -289,7 +289,7 @@ public class ZlibCompressor implements Compressor {
}
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
if (stream == 0 || b == null) {
throw new NullPointerException();
}
@@ -300,7 +300,7 @@ public class ZlibCompressor implements Compressor {
}
@Override
- public synchronized boolean needsInput() {
+ public boolean needsInput() {
// Consume remaining compressed data?
if (compressedDirectBuf.remaining() > 0) {
return false;
@@ -329,19 +329,19 @@ public class ZlibCompressor implements Compressor {
}
@Override
- public synchronized void finish() {
+ public void finish() {
finish = true;
}
@Override
- public synchronized boolean finished() {
+ public boolean finished() {
// Check if 'zlib' says its 'finished' and
// all compressed data has been consumed
return (finished && compressedDirectBuf.remaining() == 0);
}
@Override
- public synchronized int compress(byte[] b, int off, int len)
+ public int compress(byte[] b, int off, int len)
throws IOException {
if (b == null) {
throw new NullPointerException();
@@ -392,7 +392,7 @@ public class ZlibCompressor implements Compressor {
* @return the total (non-negative) number of compressed bytes output so far
*/
@Override
- public synchronized long getBytesWritten() {
+ public long getBytesWritten() {
checkStream();
return getBytesWritten(stream);
}
@@ -403,13 +403,13 @@ public class ZlibCompressor implements Compressor {
* @return the total (non-negative) number of uncompressed bytes input so far
*/
@Override
- public synchronized long getBytesRead() {
+ public long getBytesRead() {
checkStream();
return getBytesRead(stream);
}
@Override
- public synchronized void reset() {
+ public void reset() {
checkStream();
reset(stream);
finish = false;
@@ -423,7 +423,7 @@ public class ZlibCompressor implements Compressor {
}
@Override
- public synchronized void end() {
+ public void end() {
if (stream != 0) {
end(stream);
stream = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
index 575ce3c..89c879a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
@@ -120,7 +120,7 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized void setInput(byte[] b, int off, int len) {
+ public void setInput(byte[] b, int off, int len) {
if (b == null) {
throw new NullPointerException();
}
@@ -139,7 +139,7 @@ public class ZlibDecompressor implements Decompressor {
uncompressedDirectBuf.position(directBufferSize);
}
- synchronized void setInputFromSavedData() {
+ void setInputFromSavedData() {
compressedDirectBufOff = 0;
compressedDirectBufLen = userBufLen;
if (compressedDirectBufLen > directBufferSize) {
@@ -157,7 +157,7 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
if (stream == 0 || b == null) {
throw new NullPointerException();
}
@@ -169,7 +169,7 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized boolean needsInput() {
+ public boolean needsInput() {
// Consume remaining compressed data?
if (uncompressedDirectBuf.remaining() > 0) {
return false;
@@ -189,19 +189,19 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized boolean needsDictionary() {
+ public boolean needsDictionary() {
return needDict;
}
@Override
- public synchronized boolean finished() {
+ public boolean finished() {
// Check if 'zlib' says it's 'finished' and
// all compressed data has been consumed
return (finished && uncompressedDirectBuf.remaining() == 0);
}
@Override
- public synchronized int decompress(byte[] b, int off, int len)
+ public int decompress(byte[] b, int off, int len)
throws IOException {
if (b == null) {
throw new NullPointerException();
@@ -240,7 +240,7 @@ public class ZlibDecompressor implements Decompressor {
*
* @return the total (non-negative) number of uncompressed bytes output so far
*/
- public synchronized long getBytesWritten() {
+ public long getBytesWritten() {
checkStream();
return getBytesWritten(stream);
}
@@ -250,7 +250,7 @@ public class ZlibDecompressor implements Decompressor {
*
* @return the total (non-negative) number of compressed bytes input so far
*/
- public synchronized long getBytesRead() {
+ public long getBytesRead() {
checkStream();
return getBytesRead(stream);
}
@@ -263,7 +263,7 @@ public class ZlibDecompressor implements Decompressor {
* @return the total (non-negative) number of unprocessed bytes in input
*/
@Override
- public synchronized int getRemaining() {
+ public int getRemaining() {
checkStream();
return userBufLen + getRemaining(stream); // userBuf + compressedDirectBuf
}
@@ -272,7 +272,7 @@ public class ZlibDecompressor implements Decompressor {
* Resets everything including the input buffers (user and direct).</p>
*/
@Override
- public synchronized void reset() {
+ public void reset() {
checkStream();
reset(stream);
finished = false;
@@ -284,7 +284,7 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized void end() {
+ public void end() {
if (stream != 0) {
end(stream);
stream = 0;
@@ -372,7 +372,7 @@ public class ZlibDecompressor implements Decompressor {
private boolean endOfInput;
@Override
- public synchronized void decompress(ByteBuffer src, ByteBuffer dst)
+ public void decompress(ByteBuffer src, ByteBuffer dst)
throws IOException {
assert dst.isDirect() : "dst.isDirect()";
assert src.isDirect() : "src.isDirect()";
@@ -382,13 +382,13 @@ public class ZlibDecompressor implements Decompressor {
}
@Override
- public synchronized void setDictionary(byte[] b, int off, int len) {
+ public void setDictionary(byte[] b, int off, int len) {
throw new UnsupportedOperationException(
"byte[] arrays are not supported for DirectDecompressor");
}
@Override
- public synchronized int decompress(byte[] b, int off, int len) {
+ public int decompress(byte[] b, int off, int len) {
throw new UnsupportedOperationException(
"byte[] arrays are not supported for DirectDecompressor");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23cd97aa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
index 551f282..5cacebf 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
@@ -19,6 +19,18 @@ package org.apache.hadoop.io.compress;
import static org.junit.Assert.assertEquals;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.conf.Configuration;
import org.junit.Before;
import org.junit.Test;
@@ -67,4 +79,79 @@ public class TestCodecPool {
assertEquals(LEASE_COUNT_ERR, 0,
CodecPool.getLeasedDecompressorsCount(codec));
}
+
+ @Test(timeout = 1000)
+ public void testMultiThreadedCompressorPool() throws InterruptedException {
+ final int iterations = 4;
+ ExecutorService threadpool = Executors.newFixedThreadPool(3);
+ final LinkedBlockingDeque<Compressor> queue = new LinkedBlockingDeque<Compressor>(
+ 2 * iterations);
+
+ Callable<Boolean> consumer = new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ Compressor c = queue.take();
+ CodecPool.returnCompressor(c);
+ return c != null;
+ }
+ };
+
+ Callable<Boolean> producer = new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ Compressor c = CodecPool.getCompressor(codec);
+ queue.put(c);
+ return c != null;
+ }
+ };
+
+ for (int i = 0; i < iterations; i++) {
+ threadpool.submit(consumer);
+ threadpool.submit(producer);
+ }
+
+ // wait for completion
+ threadpool.shutdown();
+ threadpool.awaitTermination(1000, TimeUnit.SECONDS);
+
+ assertEquals(LEASE_COUNT_ERR, 0, CodecPool.getLeasedCompressorsCount(codec));
+ }
+
+ @Test(timeout = 1000)
+ public void testMultiThreadedDecompressorPool() throws InterruptedException {
+ final int iterations = 4;
+ ExecutorService threadpool = Executors.newFixedThreadPool(3);
+ final LinkedBlockingDeque<Decompressor> queue = new LinkedBlockingDeque<Decompressor>(
+ 2 * iterations);
+
+ Callable<Boolean> consumer = new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ Decompressor dc = queue.take();
+ CodecPool.returnDecompressor(dc);
+ return dc != null;
+ }
+ };
+
+ Callable<Boolean> producer = new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ Decompressor c = CodecPool.getDecompressor(codec);
+ queue.put(c);
+ return c != null;
+ }
+ };
+
+ for (int i = 0; i < iterations; i++) {
+ threadpool.submit(consumer);
+ threadpool.submit(producer);
+ }
+
+ // wait for completion
+ threadpool.shutdown();
+ threadpool.awaitTermination(1000, TimeUnit.SECONDS);
+
+ assertEquals(LEASE_COUNT_ERR, 0,
+ CodecPool.getLeasedDecompressorsCount(codec));
+ }
}
[15/50] [abbrv] git commit: HDFS-7122. Use of ThreadLocal
results in poor block placement. (wang)
Posted by st...@apache.org.
HDFS-7122. Use of ThreadLocal<Random> results in poor block placement. (wang)
(cherry picked from commit d7086c563ff2847c415913ac625b2a557eeccbdd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba0c76c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba0c76c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba0c76c2
Branch: refs/heads/champlain
Commit: ba0c76c24b60179f5d68860bd60905871b16b9a2
Parents: 6cb0259
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 29 14:50:28 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:14:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/net/NetworkTopology.java | 25 ++++----------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 7 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba0c76c2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 3404ff0..a11ba9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -674,27 +674,11 @@ public class NetworkTopology {
return node1.getParent()==node2.getParent();
}
- private static final ThreadLocal<Random> r = new ThreadLocal<Random>();
-
- /**
- * Getter for thread-local Random, which provides better performance than
- * a shared Random (even though Random is thread-safe).
- *
- * @return Thread-local Random.
- */
- protected Random getRandom() {
- Random rand = r.get();
- if (rand == null) {
- rand = new Random();
- r.set(rand);
- }
- return rand;
- }
+ private static final Random r = new Random();
@VisibleForTesting
void setRandomSeed(long seed) {
- Random rand = getRandom();
- rand.setSeed(seed);
+ r.setSeed(seed);
}
/** randomly choose one node from <i>scope</i>
@@ -746,7 +730,7 @@ public class NetworkTopology {
"Failed to find datanode (scope=\"" + String.valueOf(scope) +
"\" excludedScope=\"" + String.valueOf(excludedScope) + "\").");
}
- int leaveIndex = getRandom().nextInt(numOfDatanodes);
+ int leaveIndex = r.nextInt(numOfDatanodes);
return innerNode.getLeaf(leaveIndex, node);
}
@@ -919,11 +903,10 @@ public class NetworkTopology {
list.add(node);
}
- Random rand = getRandom();
int idx = 0;
for (List<Node> list: tree.values()) {
if (list != null) {
- Collections.shuffle(list, rand);
+ Collections.shuffle(list, r);
for (Node n: list) {
nodes[idx] = n;
idx++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba0c76c2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 506620f..e702277 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -301,6 +301,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6865. Byte array native checksumming on client side
(James Thomas via todd)
+ HDFS-7122. Use of ThreadLocal<Random> results in poor block placement.
+ (wang)
+
BUG FIXES
HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for
[33/50] [abbrv] git commit: HADOOP-10404. Some accesses to
DomainSocketWatcher#closed are not protected by the lock (cmccabe)
Posted by st...@apache.org.
HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected by the lock (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cbe727e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cbe727e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cbe727e
Branch: refs/heads/champlain
Commit: 3cbe727e5d3f53136c7570c59a21bde15e4e1648
Parents: ef3c0ba
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Oct 6 14:40:41 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:25:54 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/net/unix/DomainSocketWatcher.java | 17 ++++++++++++-----
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cbe727e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5db43c..db2795c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -466,6 +466,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11169. Fix DelegationTokenAuthenticatedURL to pass the connection
Configurator to the authenticator. (Arun Suresh via wang)
+ HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
+ by lock (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cbe727e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
index 8348aeb..95ef30d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
@@ -101,6 +101,7 @@ public final class DomainSocketWatcher implements Closeable {
*/
private class NotificationHandler implements Handler {
public boolean handle(DomainSocket sock) {
+ assert(lock.isHeldByCurrentThread());
try {
if (LOG.isTraceEnabled()) {
LOG.trace(this + ": NotificationHandler: doing a read on " +
@@ -346,6 +347,7 @@ public final class DomainSocketWatcher implements Closeable {
* Wake up the DomainSocketWatcher thread.
*/
private void kick() {
+ assert(lock.isHeldByCurrentThread());
try {
notificationSockets[0].getOutputStream().write(0);
} catch (IOException e) {
@@ -461,12 +463,17 @@ public final class DomainSocketWatcher implements Closeable {
} catch (IOException e) {
LOG.error(toString() + " terminating on IOException", e);
} finally {
- kick(); // allow the handler for notificationSockets[0] to read a byte
- for (Entry entry : entries.values()) {
- sendCallback("close", entries, fdSet, entry.getDomainSocket().fd);
+ lock.lock();
+ try {
+ kick(); // allow the handler for notificationSockets[0] to read a byte
+ for (Entry entry : entries.values()) {
+ sendCallback("close", entries, fdSet, entry.getDomainSocket().fd);
+ }
+ entries.clear();
+ fdSet.close();
+ } finally {
+ lock.unlock();
}
- entries.clear();
- fdSet.close();
}
}
});
[22/50] [abbrv] git commit: HDFS-6754.
TestNamenodeCapacityReport.testXceiverCount may sometimes fail due to lack of
retry. Contributed by Mit Desai. (cherry picked from commit
3f25d916d5539917092e2f52a8c2df2cfd647c3c)
Posted by st...@apache.org.
HDFS-6754. TestNamenodeCapacityReport.testXceiverCount may sometimes fail due to lack of retry. Contributed by Mit Desai.
(cherry picked from commit 3f25d916d5539917092e2f52a8c2df2cfd647c3c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b16b1a38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b16b1a38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b16b1a38
Branch: refs/heads/champlain
Commit: b16b1a38cb2b0dded1379553cfd850d6f5d61b8f
Parents: 4040cbf
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Oct 1 14:44:30 2014 -0500
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:34:14 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b16b1a38/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e719821..c70bd9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -535,6 +535,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7178. Additional unit test for replica write with full disk.
(Arpit Agarwal)
+ HDFS-6754. TestNamenodeCapacityReport may sometimes fail due to lack of
+ retry. (Mit Desai via kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b16b1a38/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
index 48a4b3c..15cad04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
@@ -169,8 +169,8 @@ public class TestNamenodeCapacityReport {
@Test
public void testXceiverCount() throws Exception {
Configuration conf = new HdfsConfiguration();
- // don't waste time retrying if close fails
- conf.setInt(DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 0);
+ // retry one time, if close fails
+ conf.setInt(DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 1);
MiniDFSCluster cluster = null;
final int nodes = 8;
[09/50] [abbrv] git commit: HDFS-4227. Document
dfs.namenode.resource.* (Daisuke Kobayashi via aw)
Posted by st...@apache.org.
HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e6e5ac0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e6e5ac0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e6e5ac0
Branch: refs/heads/champlain
Commit: 8e6e5ac05b86aabd5fd8ac823eeac5eb18270c25
Parents: 2b9e1a6
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Sep 29 09:01:59 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:57:34 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../src/main/resources/hdfs-default.xml | 38 ++++++++++++++++++++
2 files changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e6e5ac0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5afe95d..193a1f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -764,6 +764,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6664. HDFS permissions guide documentation states incorrect default
group mapping class. (Ray Chiang via aw)
+ HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e6e5ac0/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 3da0cdb..d219336 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -644,6 +644,44 @@
</property>
<property>
+ <name>dfs.namenode.resource.check.interval</name>
+ <value>5000</value>
+ <description>
+ The interval in milliseconds at which the NameNode resource checker runs.
+ The checker calculates the number of the NameNode storage volumes whose
+ available spaces are more than dfs.namenode.resource.du.reserved, and
+ enters safemode if the number becomes lower than the minimum value
+ specified by dfs.namenode.resource.checked.volumes.minimum.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.resource.du.reserved</name>
+ <value>104857600</value>
+ <description>
+ The amount of space to reserve/require for a NameNode storage directory
+ in bytes. The default is 100MB.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.resource.checked.volumes</name>
+ <value></value>
+ <description>
+ A list of local directories for the NameNode resource checker to check in
+ addition to the local edits directories.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.resource.checked.volumes.minimum</name>
+ <value>1</value>
+ <description>
+ The minimum number of redundant NameNode storage volumes required.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.balance.bandwidthPerSec</name>
<value>1048576</value>
<description>
[10/50] [abbrv] git commit: HDFS-4165. Faulty sanity check in
FsDirectory.unprotectedSetQuota. Contributed by Binglin Chang.
Posted by st...@apache.org.
HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota. Contributed by Binglin Chang.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1407245 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 0f7a4337ac96141db4a3400fd8308436d87042f4)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33739588
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33739588
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33739588
Branch: refs/heads/champlain
Commit: 33739588a6ddcd77c5d829225a184b199582b006
Parents: 8e6e5ac
Author: Suresh Srinivas <su...@apache.org>
Authored: Thu Nov 8 19:54:55 2012 +0000
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:05:05 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33739588/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 193a1f5..fdc1153 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -284,6 +284,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6519. Document oiv_legacy command (Akira AJISAKA via aw)
+ HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota.
+ (Binglin Chang via suresh)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33739588/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 0ad4754..c817b88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2264,9 +2264,9 @@ public class FSDirectory implements Closeable {
assert hasWriteLock();
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
- nsQuota < HdfsConstants.QUOTA_RESET) ||
+ nsQuota != HdfsConstants.QUOTA_RESET) ||
(dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET &&
- dsQuota < HdfsConstants.QUOTA_RESET)) {
+ dsQuota != HdfsConstants.QUOTA_RESET)) {
throw new IllegalArgumentException("Illegal value for nsQuota or " +
"dsQuota : " + nsQuota + " and " +
dsQuota);
[12/50] [abbrv] git commit: HDFS-7104. Fix and clarify INodeInPath
getter functions. Contributed by Zhe Zhang.
Posted by st...@apache.org.
HDFS-7104. Fix and clarify INodeInPath getter functions. Contributed by Zhe Zhang.
(cherry picked from commit f0293f11a85176afc04d809574c9126a80b700cc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38ed6949
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38ed6949
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38ed6949
Branch: refs/heads/champlain
Commit: 38ed69493f2a8a16cd3f4da8315617bc93e32148
Parents: b5882dd
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 29 12:58:37 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:08:51 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hdfs/server/namenode/INodesInPath.java | 25 ++++++++++++--------
.../server/namenode/TestSnapshotPathINodes.java | 6 ++---
3 files changed, 20 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38ed6949/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fdc1153..25bfbb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -287,6 +287,8 @@ Release 2.6.0 - UNRELEASED
HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota.
(Binglin Chang via suresh)
+ HDFS-7104. Fix and clarify INodeInPath getter functions. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38ed6949/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index ba052a4..c74ebb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -133,7 +133,6 @@ public class INodesInPath {
* be thrown when the path refers to a symbolic link.
* @return the specified number of existing INodes in the path
*/
- // TODO: Eliminate null elements from inodes (to be provided by HDFS-7104)
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final int numOfINodes,
final boolean resolveLink) throws UnresolvedLinkException {
@@ -262,7 +261,8 @@ public class INodesInPath {
*/
private boolean isSnapshot;
/**
- * Index of {@link INodeDirectoryWithSnapshot} for snapshot path, else -1
+ * index of the {@link Snapshot.Root} node in the inodes array,
+ * -1 for non-snapshot paths.
*/
private int snapshotRootIndex;
/**
@@ -312,15 +312,20 @@ public class INodesInPath {
}
/**
- * @return the inodes array excluding the null elements.
+ * @return a new array of inodes excluding the null elements introduced by
+ * snapshot path elements. E.g., after resolving path "/dir/.snapshot",
+ * {@link #inodes} is {/, dir, null}, while the returned array only contains
+ * inodes of "/" and "dir". Note the length of the returned array is always
+ * equal to {@link #capacity}.
*/
INode[] getINodes() {
- if (capacity < inodes.length) {
- INode[] newNodes = new INode[capacity];
- System.arraycopy(inodes, 0, newNodes, 0, capacity);
- inodes = newNodes;
+ if (capacity == inodes.length) {
+ return inodes;
}
- return inodes;
+
+ INode[] newNodes = new INode[capacity];
+ System.arraycopy(inodes, 0, newNodes, 0, capacity);
+ return newNodes;
}
/**
@@ -341,8 +346,8 @@ public class INodesInPath {
}
/**
- * @return index of the {@link INodeDirectoryWithSnapshot} in
- * {@link #inodes} for snapshot path, else -1.
+ * @return index of the {@link Snapshot.Root} node in the inodes array,
+ * -1 for non-snapshot paths.
*/
int getSnapshotRootIndex() {
return this.snapshotRootIndex;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38ed6949/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
index f9fb1cf..95524e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java
@@ -215,7 +215,7 @@ public class TestSnapshotPathINodes {
// snapshotRootIndex should be -1.
assertSnapshot(nodesInPath, true, snapshot, -1);
// Check the INode for file1 (snapshot file)
- assertINodeFile(nodesInPath.getLastINode(), file1);
+ assertINodeFile(inodes[inodes.length - 1], file1);
// Call getExistingPathINodes and request 2 INodes.
nodesInPath = INodesInPath.resolve(fsdir.rootDir, components, 2, false);
@@ -224,7 +224,7 @@ public class TestSnapshotPathINodes {
// There should be two INodes in inodes: s1 and snapshot of file1. Thus the
// SnapshotRootIndex should be 0.
assertSnapshot(nodesInPath, true, snapshot, 0);
- assertINodeFile(nodesInPath.getLastINode(), file1);
+ assertINodeFile(inodes[inodes.length - 1], file1);
// Resolve the path "/TestSnapshot/sub1/.snapshot"
String dotSnapshotPath = sub1.toString() + "/.snapshot";
@@ -239,7 +239,7 @@ public class TestSnapshotPathINodes {
// No SnapshotRoot dir is included in the resolved inodes
assertSnapshot(nodesInPath, true, snapshot, -1);
// The last INode should be the INode for sub1
- final INode last = nodesInPath.getLastINode();
+ final INode last = inodes[inodes.length - 1];
assertEquals(last.getFullPathName(), sub1.toString());
assertFalse(last instanceof INodeFile);
[04/50] [abbrv] git commit: HADOOP-10552. Fix usage and example at
FileSystemShell.apt.vm (Kenji Kikushima via aw)
Posted by st...@apache.org.
HADOOP-10552. Fix usage and example at FileSystemShell.apt.vm (Kenji Kikushima via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4389c01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4389c01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4389c01
Branch: refs/heads/champlain
Commit: d4389c0131667a3717376e5dd70ea58fb1647d93
Parents: d37969b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 26 14:57:20 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:48:48 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/apt/FileSystemShell.apt.vm | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4389c01/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index adc92c5d..62f9431 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -564,6 +564,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11048. user/custom LogManager fails to load if the client
classloader is enabled (Sangjin Lee via jlowe)
+ HADOOP-10552. Fix usage and example at FileSystemShell.apt.vm
+ (Kenji Kikushima via aw)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4389c01/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
index 9317c66..abc4643 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
@@ -370,7 +370,7 @@ mkdir
moveFromLocal
- Usage: <<<dfs -moveFromLocal <localsrc> <dst> >>>
+ Usage: <<<hdfs dfs -moveFromLocal <localsrc> <dst> >>>
Similar to put command, except that the source localsrc is deleted after
it's copied.
@@ -617,7 +617,7 @@ touchz
Example:
- * <<<hadoop -touchz pathname>>>
+ * <<<hdfs dfs -touchz pathname>>>
Exit Code:
Returns 0 on success and -1 on error.
[26/50] [abbrv] git commit: HDFS-7162. Wrong path when deleting
through fuse-dfs a file which already exists in trash (cmccabe) (cherry
picked from commit 03db9cc839663cad387db7df8e4f60b312058f18) (cherry picked
from commit ab1afc4e1cafc1cd81cca36eccc1c9
Posted by st...@apache.org.
HDFS-7162. Wrong path when deleting through fuse-dfs a file which already exists in trash (cmccabe)
(cherry picked from commit 03db9cc839663cad387db7df8e4f60b312058f18)
(cherry picked from commit ab1afc4e1cafc1cd81cca36eccc1c905afb7293b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54465f7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54465f7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54465f7e
Branch: refs/heads/champlain
Commit: 54465f7e38a6e3cb0a1de5ac20d6fde491b61175
Parents: c0cf126
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Oct 2 12:42:19 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:58:25 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c | 7 ++++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54465f7e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6427c80..7a33eeb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -547,6 +547,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7181. Remove incorrect precondition check on key length in
FileEncryptionInfo. (wang)
+ HDFS-7162. Wrong path when deleting through fuse-dfs a file which already
+ exists in trash (Chenging Liu via cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54465f7e/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
index 688eae8..5e58087 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_trash.c
@@ -150,13 +150,14 @@ int move_to_trash(const char *abs_path, hdfsFS userFS)
if (ret) {
goto done;
}
- if (!strncmp(trash_base, abs_path, strlen(trash_base))) {
+ int trash_base_len = strlen(trash_base);
+ if (!strncmp(trash_base, abs_path, trash_base_len)
+ && (strlen(abs_path) == trash_base_len || abs_path[trash_base_len] == '/')) {
INFO("move_to_trash(%s): file is already in the trash; deleting.",
abs_path);
ret = ALREADY_IN_TRASH_ERR;
goto done;
}
- fprintf(stderr, "trash_base='%s'\n", trash_base);
if (asprintf(&target_dir, "%s%s", trash_base, parent_dir) < 0) {
ret = ENOMEM;
target_dir = NULL;
@@ -182,7 +183,7 @@ int move_to_trash(const char *abs_path, hdfsFS userFS)
int idx;
for (idx = 1; idx < TRASH_RENAME_TRIES; idx++) {
free(target);
- if (asprintf(&target, "%s%s.%d", target_dir, pcomp, idx) < 0) {
+ if (asprintf(&target, "%s/%s.%d", target_dir, pcomp, idx) < 0) {
target = NULL;
ret = ENOMEM;
goto done;
[13/50] [abbrv] git commit: HADOOP-11110. JavaKeystoreProvider should
not report a key as created if it was not flushed to the backing file. (Arun
Suresh via wang)
Posted by st...@apache.org.
HADOOP-11110. JavaKeystoreProvider should not report a key as created if it was not flushed to the backing file. (Arun Suresh via wang)
(cherry picked from commit a78953c974e52abe73905b1901a2354696f4a5a0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ce35f80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ce35f80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ce35f80
Branch: refs/heads/champlain
Commit: 2ce35f80101a3bdd1a0737319ec5f0219a19e7b5
Parents: 38ed694
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 29 13:10:26 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:12:26 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 58 +++++++++++---
.../org/apache/hadoop/crypto/key/KeyShell.java | 6 +-
.../FailureInjectingJavaKeyStoreProvider.java | 80 ++++++++++++++++++++
.../crypto/key/TestKeyProviderFactory.java | 48 +++++++++++-
....apache.hadoop.crypto.key.KeyProviderFactory | 19 +++++
6 files changed, 201 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f3ef5ab..cabaadc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -438,6 +438,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7157. Using Time.now() for recording start/end time of reconfiguration
tasks (Lei Xu via cmccabe)
+ HADOOP-1110. JavaKeystoreProvider should not report a key as created if it
+ was not flushed to the backing file.
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 30583eb..5cc136c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -30,6 +31,8 @@ import org.apache.hadoop.security.ProviderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
@@ -107,6 +110,20 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
+ @VisibleForTesting
+ JavaKeyStoreProvider(JavaKeyStoreProvider other) {
+ super(new Configuration());
+ uri = other.uri;
+ path = other.path;
+ fs = other.fs;
+ permissions = other.permissions;
+ keyStore = other.keyStore;
+ password = other.password;
+ changed = other.changed;
+ readLock = other.readLock;
+ writeLock = other.writeLock;
+ }
+
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
super(conf);
this.uri = uri;
@@ -501,6 +518,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
public void flush() throws IOException {
Path newPath = constructNewPath(path);
Path oldPath = constructOldPath(path);
+ Path resetPath = path;
writeLock.lock();
try {
if (!changed) {
@@ -527,6 +545,9 @@ public class JavaKeyStoreProvider extends KeyProvider {
// Save old File first
boolean fileExisted = backupToOld(oldPath);
+ if (fileExisted) {
+ resetPath = oldPath;
+ }
// write out the keystore
// Write to _NEW path first :
try {
@@ -534,16 +555,34 @@ public class JavaKeyStoreProvider extends KeyProvider {
} catch (IOException ioe) {
// rename _OLD back to curent and throw Exception
revertFromOld(oldPath, fileExisted);
+ resetPath = path;
throw ioe;
}
// Rename _NEW to CURRENT and delete _OLD
cleanupNewAndOld(newPath, oldPath);
changed = false;
+ } catch (IOException ioe) {
+ resetKeyStoreState(resetPath);
+ throw ioe;
} finally {
writeLock.unlock();
}
}
+ private void resetKeyStoreState(Path path) {
+ LOG.debug("Could not flush Keystore.."
+ + "attempting to reset to previous state !!");
+ // 1) flush cache
+ cache.clear();
+ // 2) load keyStore from previous path
+ try {
+ loadFromPath(path, password);
+ LOG.debug("KeyStore resetting to previously flushed state !!");
+ } catch (Exception e) {
+ LOG.debug("Could not reset Keystore to previous state", e);
+ }
+ }
+
private void cleanupNewAndOld(Path newPath, Path oldPath) throws IOException {
// Rename _NEW to CURRENT
renameOrFail(newPath, path);
@@ -553,7 +592,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
}
}
- private void writeToNew(Path newPath) throws IOException {
+ protected void writeToNew(Path newPath) throws IOException {
FSDataOutputStream out =
FileSystem.create(fs, newPath, permissions);
try {
@@ -570,14 +609,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
out.close();
}
- private void revertFromOld(Path oldPath, boolean fileExisted)
- throws IOException {
- if (fileExisted) {
- renameOrFail(oldPath, path);
- }
- }
-
- private boolean backupToOld(Path oldPath)
+ protected boolean backupToOld(Path oldPath)
throws IOException {
boolean fileExisted = false;
if (fs.exists(path)) {
@@ -587,6 +619,14 @@ public class JavaKeyStoreProvider extends KeyProvider {
return fileExisted;
}
+ private void revertFromOld(Path oldPath, boolean fileExisted)
+ throws IOException {
+ if (fileExisted) {
+ renameOrFail(oldPath, path);
+ }
+ }
+
+
private void renameOrFail(Path src, Path dest)
throws IOException {
if (!fs.rename(src, dest)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index b0c427a..3875125 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -345,8 +345,8 @@ public class KeyShell extends Configured implements Tool {
+ provider + "\n for key name: " + keyName);
try {
provider.rollNewVersion(keyName);
- out.println(keyName + " has been successfully rolled.");
provider.flush();
+ out.println(keyName + " has been successfully rolled.");
printProviderWritten();
} catch (NoSuchAlgorithmException e) {
out.println("Cannot roll key: " + keyName + " within KeyProvider: "
@@ -418,8 +418,8 @@ public class KeyShell extends Configured implements Tool {
if (cont) {
try {
provider.deleteKey(keyName);
- out.println(keyName + " has been successfully deleted.");
provider.flush();
+ out.println(keyName + " has been successfully deleted.");
printProviderWritten();
} catch (IOException e) {
out.println(keyName + " has not been deleted.");
@@ -479,9 +479,9 @@ public class KeyShell extends Configured implements Tool {
warnIfTransientProvider();
try {
provider.createKey(keyName, options);
+ provider.flush();
out.println(keyName + " has been successfully created with options "
+ options.toString() + ".");
- provider.flush();
printProviderWritten();
} catch (InvalidParameterException e) {
out.println(keyName + " has not been created. " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/FailureInjectingJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/FailureInjectingJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/FailureInjectingJavaKeyStoreProvider.java
new file mode 100644
index 0000000..8b41c45
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/FailureInjectingJavaKeyStoreProvider.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+public class FailureInjectingJavaKeyStoreProvider extends JavaKeyStoreProvider {
+
+ public static final String SCHEME_NAME = "failjceks";
+
+ private boolean backupFail = false;
+ private boolean writeFail = false;
+ FailureInjectingJavaKeyStoreProvider(JavaKeyStoreProvider prov) {
+ super(prov);
+ }
+
+ public void setBackupFail(boolean b) {
+ backupFail = b;
+ }
+
+ public void setWriteFail(boolean b) {
+ backupFail = b;
+ }
+
+ // Failure injection methods..
+ @Override
+ public void writeToNew(Path newPath) throws IOException {
+ if (writeFail) {
+ throw new IOException("Injecting failure on write");
+ }
+ super.writeToNew(newPath);
+ }
+
+ @Override
+ public boolean backupToOld(Path oldPath) throws IOException {
+ if (backupFail) {
+ throw new IOException("Inejection Failure on backup");
+ }
+ return super.backupToOld(oldPath);
+ }
+
+ public static class Factory extends KeyProviderFactory {
+ @Override
+ public KeyProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ try {
+ return new FailureInjectingJavaKeyStoreProvider(
+ (JavaKeyStoreProvider) new JavaKeyStoreProvider.Factory()
+ .createProvider(
+ new URI(providerName.toString().replace(SCHEME_NAME,
+ JavaKeyStoreProvider.SCHEME_NAME)), conf));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
index 8fb661d..7bb12d0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
@@ -40,6 +40,7 @@ import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
public class TestKeyProviderFactory {
@@ -171,6 +172,7 @@ public class TestKeyProviderFactory {
assertEquals("Key no-such-key not found", e.getMessage());
}
provider.flush();
+
// get a new instance of the provider to ensure it was saved correctly
provider = KeyProviderFactory.getProviders(conf).get(0);
assertArrayEquals(new byte[]{2},
@@ -215,6 +217,50 @@ public class TestKeyProviderFactory {
file.delete();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
checkSpecificProvider(conf, ourUrl);
+
+ // START : Test flush error by failure injection
+ conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl.replace(
+ JavaKeyStoreProvider.SCHEME_NAME,
+ FailureInjectingJavaKeyStoreProvider.SCHEME_NAME));
+ // get a new instance of the provider to ensure it was saved correctly
+ KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+ // inject failure during keystore write
+ FailureInjectingJavaKeyStoreProvider fProvider =
+ (FailureInjectingJavaKeyStoreProvider) provider;
+ fProvider.setWriteFail(true);
+ provider.createKey("key5", new byte[]{1},
+ KeyProvider.options(conf).setBitLength(8));
+ assertNotNull(provider.getCurrentKey("key5"));
+ try {
+ provider.flush();
+ Assert.fail("Should not succeed");
+ } catch (Exception e) {
+ // Ignore
+ }
+ // SHould be reset to pre-flush state
+ Assert.assertNull(provider.getCurrentKey("key5"));
+
+ // Un-inject last failure and
+ // inject failure during keystore backup
+ fProvider.setWriteFail(false);
+ fProvider.setBackupFail(true);
+ provider.createKey("key6", new byte[]{1},
+ KeyProvider.options(conf).setBitLength(8));
+ assertNotNull(provider.getCurrentKey("key6"));
+ try {
+ provider.flush();
+ Assert.fail("Should not succeed");
+ } catch (Exception e) {
+ // Ignore
+ }
+ // SHould be reset to pre-flush state
+ Assert.assertNull(provider.getCurrentKey("key6"));
+ // END : Test flush error by failure injection
+
+ conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl.replace(
+ FailureInjectingJavaKeyStoreProvider.SCHEME_NAME,
+ JavaKeyStoreProvider.SCHEME_NAME));
+
Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
@@ -227,7 +273,7 @@ public class TestKeyProviderFactory {
file.delete();
file.createNewFile();
assertTrue(oldFile.exists());
- KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+ provider = KeyProviderFactory.getProviders(conf).get(0);
assertTrue(file.exists());
assertTrue(oldFile + "should be deleted", !oldFile.exists());
verifyAfterReload(file, provider);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ce35f80/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
new file mode 100644
index 0000000..74b49a0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.crypto.key.JavaKeyStoreProvider$Factory
+org.apache.hadoop.crypto.key.UserProvider$Factory
+org.apache.hadoop.crypto.key.kms.KMSClientProvider$Factory
+org.apache.hadoop.crypto.key.FailureInjectingJavaKeyStoreProvider$Factory
[44/50] [abbrv] git commit: MAPREDUCE-6122. Fix CHANGES.txt.
Posted by st...@apache.org.
MAPREDUCE-6122. Fix CHANGES.txt.
(cherry picked from commit 2f0166db560b180e8d07c10ba9ad4bfaf2aa7b36)
(cherry picked from commit 61be28f501ba3de0915c165c9d43fa38463722c0)
(cherry picked from commit 4573bbe33d7016604787f1f4999fadc057da1b5e)
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75e42301
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75e42301
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75e42301
Branch: refs/heads/champlain
Commit: 75e42301491b8da2995f481a68899fd55a137d68
Parents: 45e6bd5
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 22:32:28 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 22:55:02 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75e42301/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index cbc71b2..a75d8a8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -192,7 +192,7 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
(Akira AJISAKA via jlowe)
- MAPREDUCE-6029. TestLineRecordReader may fail due to test data files checked
+ MAPREDUCE-6122. TestLineRecordReader may fail due to test data files checked
out of git with incorrect line endings. (cnauroth)
Release 2.5.1 - 2014-09-05
[05/50] [abbrv] git commit: HADOOP-11143 NetUtils.wrapException loses
inner stack trace on BindException
Posted by st...@apache.org.
HADOOP-11143 NetUtils.wrapException loses inner stack trace on BindException
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae1634f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae1634f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae1634f7
Branch: refs/heads/champlain
Commit: ae1634f7489708c17d608e11bda61e212e3b39f3
Parents: d4389c0
Author: Steve Loughran <st...@apache.org>
Authored: Sat Sep 27 21:06:31 2014 +0100
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:49:15 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae1634f7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 62f9431..922bae8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -567,6 +567,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10552. Fix usage and example at FileSystemShell.apt.vm
(Kenji Kikushima via aw)
+ HADOOP-11143 NetUtils.wrapException loses inner stack trace on BindException
+ (stevel)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae1634f7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 8595f87..9ee0f3e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -716,7 +716,7 @@ public class NetUtils {
final int localPort,
final IOException exception) {
if (exception instanceof BindException) {
- return new BindException(
+ return wrapWithMessage(exception,
"Problem binding to ["
+ localHost
+ ":"
[17/50] [abbrv] git commit: HADOOP-11130. NFS updateMaps OS check is
reversed. Contributed by Brandon Li
Posted by st...@apache.org.
HADOOP-11130. NFS updateMaps OS check is reversed. Contributed by Brandon Li
(cherry picked from commit c88c6c57aa79e8a1f33d2e075491656f472c858b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ceea841
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ceea841
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ceea841
Branch: refs/heads/champlain
Commit: 4ceea841e20aecb857877b3523b25353c633a83c
Parents: b45b45b
Author: Brandon Li <br...@apache.org>
Authored: Mon Sep 29 15:05:43 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:24:47 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java | 15 +++++++--------
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ceea841/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e174903..17fea7f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -443,6 +443,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-1110. JavaKeystoreProvider should not report a key as created if it
was not flushed to the backing file.
+ HADOOP-11130. NFS updateMaps OS check is reversed (brandonli)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ceea841/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
index dd580cf..b037413 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/IdUserGroup.java
@@ -45,8 +45,8 @@ public class IdUserGroup {
private final static String OS = System.getProperty("os.name");
/** Shell commands to get users and groups */
- static final String LINUX_GET_ALL_USERS_CMD = "getent passwd | cut -d: -f1,3";
- static final String LINUX_GET_ALL_GROUPS_CMD = "getent group | cut -d: -f1,3";
+ static final String GET_ALL_USERS_CMD = "getent passwd | cut -d: -f1,3";
+ static final String GET_ALL_GROUPS_CMD = "getent group | cut -d: -f1,3";
static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID";
static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID";
@@ -223,16 +223,15 @@ public class IdUserGroup {
+ "' does not exist.");
}
- if (OS.startsWith("Linux")) {
- updateMapInternal(uMap, "user", LINUX_GET_ALL_USERS_CMD, ":",
+ if (OS.startsWith("Mac")) {
+ updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
staticMapping.uidMapping);
- updateMapInternal(gMap, "group", LINUX_GET_ALL_GROUPS_CMD, ":",
+ updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
staticMapping.gidMapping);
} else {
- // Mac
- updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
+ updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":",
staticMapping.uidMapping);
- updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
+ updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":",
staticMapping.gidMapping);
}
[19/50] [abbrv] git commit: HADOOP-11154. Update BUILDING.txt to
state that CMake 3.0 or newer is required on Mac. Contributed by Chris
Nauroth.
Posted by st...@apache.org.
HADOOP-11154. Update BUILDING.txt to state that CMake 3.0 or newer is required on Mac. Contributed by Chris Nauroth.
(cherry picked from commit 8dc4e9408f4cd9a50cd58aee574f3b03c3a33b31)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e327b495
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e327b495
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e327b495
Branch: refs/heads/champlain
Commit: e327b495d6ea9a329d555dfce1185aaccedb5b0c
Parents: a0fdda8
Author: cnauroth <cn...@apache.org>
Authored: Tue Sep 30 08:30:44 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:26:23 2014 -0700
----------------------------------------------------------------------
BUILDING.txt | 2 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e327b495/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index bbad5ef..621a221 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -8,7 +8,7 @@ Requirements:
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
-* CMake 2.6 or newer (if compiling native code)
+* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac
* Zlib devel (if compiling native code)
* openssl devel ( if compiling native hadoop-pipes )
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e327b495/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 17fea7f..0cbc53c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -583,6 +583,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11049. javax package system class default is too broad (Sangjin Lee
via jlowe)
+ HADOOP-11154. Update BUILDING.txt to state that CMake 3.0 or newer is
+ required on Mac. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
[25/50] [abbrv] git commit: HDFS-7181. Remove incorrect precondition
check on key length in FileEncryptionInfo. (wang)
Posted by st...@apache.org.
HDFS-7181. Remove incorrect precondition check on key length in FileEncryptionInfo. (wang)
(cherry picked from commit 6ac10516e7fa28384b6d3c2670f6621e2666ffdb)
(cherry picked from commit 6dbba19469d698891e9c6a8513cbe58992a736b1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0cf126a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0cf126a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0cf126a
Branch: refs/heads/champlain
Commit: c0cf126a27c517f664a2b8ea410fbc93a6c1c5d3
Parents: 3ebbc99
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 2 13:51:08 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:37:54 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/crypto/CipherSuite.java | 10 ----------
.../java/org/apache/hadoop/fs/FileEncryptionInfo.java | 2 --
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
3 files changed, 3 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0cf126a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
index 9962b38..c9355d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
@@ -73,16 +73,6 @@ public enum CipherSuite {
return builder.toString();
}
- public static void checkName(String name) {
- CipherSuite[] suites = CipherSuite.values();
- for (CipherSuite suite : suites) {
- if (suite.getName().equals(name)) {
- return;
- }
- }
- throw new IllegalArgumentException("Invalid cipher suite name: " + name);
- }
-
/**
* Convert to CipherSuite from name, {@link #algoBlockSize} is fixed for
* certain cipher suite, just need to compare the name.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0cf126a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
index 27e0c85..00ddfe8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -58,8 +58,6 @@ public class FileEncryptionInfo {
checkNotNull(iv);
checkNotNull(keyName);
checkNotNull(ezKeyVersionName);
- checkArgument(edek.length == suite.getAlgorithmBlockSize(),
- "Unexpected key length");
checkArgument(iv.length == suite.getAlgorithmBlockSize(),
"Unexpected IV length");
this.cipherSuite = suite;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0cf126a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cf39e25..6427c80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -544,6 +544,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7179. DFSClient should instantiate a KeyProvider, not a
KeyProviderCryptoExtension. (wang)
+ HDFS-7181. Remove incorrect precondition check on key length in
+ FileEncryptionInfo. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
[41/50] [abbrv] HADOOP-11184. Update Hadoop's lz4 to r123 (cmccabe)
(cherry picked from commit d8d628d1e0c42a32e2079ed430e8817996bbe4c4) (cherry
picked from commit 8addbe2b8eaf9fabbb0b6dc0dd3351a0145e0cae)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h
deleted file mode 100644
index fe4de1c..0000000
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- LZ4 Encoder - Part of LZ4 compression algorithm
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-/* lz4_encoder.h must be included into lz4.c
- The objective of this file is to create a single LZ4 compression function source
- which will be instanciated multiple times with minor variations
- depending on a set of #define.
-*/
-
-
-
-//****************************
-// Check required defines
-//****************************
-
-#ifndef FUNCTION_NAME
-# error "FUNTION_NAME is not defined"
-#endif
-
-
-//****************************
-// Local definitions
-//****************************
-
-#ifdef COMPRESS_64K
-# define HASHLOG (MEMORY_USAGE-1)
-# define CURRENT_H_TYPE U16
-# define CURRENTBASE(base) const BYTE* const base = ip
-#else
-# define HASHLOG (MEMORY_USAGE-2)
-# define CURRENT_H_TYPE HTYPE
-# define CURRENTBASE(base) INITBASE(base)
-#endif
-
-#define HASHTABLE_NBCELLS (1U<<HASHLOG)
-#define LZ4_HASH(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASHLOG))
-#define LZ4_HASHVALUE(p) LZ4_HASH(A32(p))
-
-
-
-//****************************
-// Function code
-//****************************
-
-int FUNCTION_NAME(
-#ifdef USE_HEAPMEMORY
- void* ctx,
-#endif
- const char* source,
- char* dest,
- int inputSize
-#ifdef LIMITED_OUTPUT
- ,int maxOutputSize
-#endif
- )
-{
-#ifdef USE_HEAPMEMORY
- CURRENT_H_TYPE* HashTable = (CURRENT_H_TYPE*)ctx;
-#else
- CURRENT_H_TYPE HashTable[HASHTABLE_NBCELLS] = {0};
-#endif
-
- const BYTE* ip = (BYTE*) source;
- CURRENTBASE(base);
- const BYTE* anchor = ip;
- const BYTE* const iend = ip + inputSize;
- const BYTE* const mflimit = iend - MFLIMIT;
-#define matchlimit (iend - LASTLITERALS)
-
- BYTE* op = (BYTE*) dest;
-#ifdef LIMITED_OUTPUT
- BYTE* const oend = op + maxOutputSize;
-#endif
-
- int length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
-
- // Init
- if (inputSize<MINLENGTH) goto _last_literals;
-#ifdef COMPRESS_64K
- if (inputSize>=LZ4_64KLIMIT) return 0; // Size too large (not within 64K limit)
-#endif
-#ifdef USE_HEAPMEMORY
- memset((void*)HashTable, 0, HASHTABLESIZE);
-#endif
-
- // First Byte
- HashTable[LZ4_HASHVALUE(ip)] = (CURRENT_H_TYPE)(ip - base);
- ip++; forwardH = LZ4_HASHVALUE(ip);
-
- // Main Loop
- for ( ; ; )
- {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE* forwardIp = ip;
- const BYTE* ref;
- BYTE* token;
-
- // Find a match
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if unlikely(forwardIp > mflimit) { goto _last_literals; }
-
- forwardH = LZ4_HASHVALUE(forwardIp);
- ref = base + HashTable[h];
- HashTable[h] = (CURRENT_H_TYPE)(ip - base);
-
- } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
-
- // Catch up
- while ((ip>anchor) && (ref>(BYTE*)source) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; }
-
- // Encode Literal length
- length = (int)(ip - anchor);
- token = op++;
-#ifdef LIMITED_OUTPUT
- if unlikely(op + length + (2 + 1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit
-#endif
- if (length>=(int)RUN_MASK)
- {
- int len = length-RUN_MASK;
- *token=(RUN_MASK<<ML_BITS);
- for(; len >= 255 ; len-=255) *op++ = 255;
- *op++ = (BYTE)len;
- }
- else *token = (BYTE)(length<<ML_BITS);
-
- // Copy Literals
- LZ4_BLINDCOPY(anchor, op, length);
-
-_next_match:
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(op,(U16)(ip-ref));
-
- // Start Counting
- ip+=MINMATCH; ref+=MINMATCH; // MinMatch already verified
- anchor = ip;
- while likely(ip<matchlimit-(STEPSIZE-1))
- {
- size_t diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) { ip+=STEPSIZE; ref+=STEPSIZE; continue; }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
- if (LZ4_ARCH64) if ((ip<(matchlimit-3)) && (A32(ref) == A32(ip))) { ip+=4; ref+=4; }
- if ((ip<(matchlimit-1)) && (A16(ref) == A16(ip))) { ip+=2; ref+=2; }
- if ((ip<matchlimit) && (*ref == *ip)) ip++;
-_endCount:
-
- // Encode MatchLength
- length = (int)(ip - anchor);
-#ifdef LIMITED_OUTPUT
- if unlikely(op + (1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit
-#endif
- if (length>=(int)ML_MASK)
- {
- *token += ML_MASK;
- length -= ML_MASK;
- for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; }
- if (length >= 255) { length-=255; *op++ = 255; }
- *op++ = (BYTE)length;
- }
- else *token += (BYTE)(length);
-
- // Test end of chunk
- if (ip > mflimit) { anchor = ip; break; }
-
- // Fill table
- HashTable[LZ4_HASHVALUE(ip-2)] = (CURRENT_H_TYPE)(ip - 2 - base);
-
- // Test next position
- ref = base + HashTable[LZ4_HASHVALUE(ip)];
- HashTable[LZ4_HASHVALUE(ip)] = (CURRENT_H_TYPE)(ip - base);
- if ((ref >= ip - MAX_DISTANCE) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; }
-
- // Prepare next loop
- anchor = ip++;
- forwardH = LZ4_HASHVALUE(ip);
- }
-
-_last_literals:
- // Encode Last Literals
- {
- int lastRun = (int)(iend - anchor);
-#ifdef LIMITED_OUTPUT
- if (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) return 0; // Check output limit
-#endif
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (BYTE)(lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-
-//****************************
-// Clean defines
-//****************************
-
-// Required defines
-#undef FUNCTION_NAME
-
-// Locally Generated
-#undef HASHLOG
-#undef HASHTABLE_NBCELLS
-#undef LZ4_HASH
-#undef LZ4_HASHVALUE
-#undef CURRENT_H_TYPE
-#undef CURRENTBASE
-
-// Optional defines
-#ifdef LIMITED_OUTPUT
-#undef LIMITED_OUTPUT
-#endif
-
-#ifdef USE_HEAPMEMORY
-#undef USE_HEAPMEMORY
-#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c
index 087ff78..34a6173 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c
@@ -1,6 +1,6 @@
/*
LZ4 HC - High Compression Mode of LZ4
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -31,36 +31,45 @@
- LZ4 source repository : http://code.google.com/p/lz4/
*/
-/*
-Note : this source file requires "lz4hc_encoder.h"
-*/
-//**************************************
-// Memory routines
-//**************************************
-#include <stdlib.h> // calloc, free
+/**************************************
+ Tuning Parameter
+**************************************/
+#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8
+
+
+/**************************************
+ Memory routines
+**************************************/
+#include <stdlib.h> /* calloc, free */
#define ALLOCATOR(s) calloc(1,s)
#define FREEMEM free
-#include <string.h> // memset, memcpy
+#include <string.h> /* memset, memcpy */
#define MEM_INIT memset
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
+/**************************************
+ CPU Feature Detection
+**************************************/
+/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+ || defined(__64BIT__) || defined(__mips64) \
+ || defined(__powerpc64__) || defined(__powerpc64le__) \
+ || defined(__ppc64__) || defined(__ppc64le__) \
+ || defined(__PPC64__) || defined(__PPC64LE__) \
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \
+ || defined(__s390x__) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
# define LZ4_ARCH64 0
#endif
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
+/*
+ * Little Endian or Big Endian ?
+ * Overwrite the #define below if you know your architecture endianess
+ */
+#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)
@@ -74,69 +83,71 @@ Note : this source file requires "lz4hc_encoder.h"
|| defined(_MIPSEB) || defined(__s390__)
# define LZ4_BIG_ENDIAN 1
#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */
#endif
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+/*
+ * Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+ */
#if defined(__ARM_FEATURE_UNALIGNED)
# define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+/* Define this parameter if your target system or compiler does not support hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LZ4_FORCE_SW_BITCOUNT
#endif
-//**************************************
-// Compiler Options
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
- /* "restrict" is a known keyword */
+/**************************************
+ Compiler Options
+**************************************/
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
+/* "restrict" is a known keyword */
#else
-# define restrict // Disable restrict
+# define restrict /* Disable restrict */
#endif
-#ifdef _MSC_VER // Visual Studio
-# define forceinline static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
+#ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# include <intrin.h> /* For Visual 2005 */
+# if LZ4_ARCH64 /* 64-bits */
+# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */
+# else /* 32-bits */
+# pragma intrinsic(_BitScanForward) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */
# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-# pragma warning(disable : 4701) // disable: C4701: potentially uninitialized local variable used
-#else
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */
+#else
# ifdef __GNUC__
-# define forceinline static inline __attribute__((always_inline))
+# define FORCE_INLINE static inline __attribute__((always_inline))
# else
-# define forceinline static inline
+# define FORCE_INLINE static inline
# endif
#endif
-#ifdef _MSC_VER // Visual Studio
+#ifdef _MSC_VER /* Visual Studio */
# define lz4_bswap16(x) _byteswap_ushort(x)
#else
# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
#endif
-//**************************************
-// Includes
-//**************************************
+/**************************************
+ Includes
+**************************************/
#include "lz4hc.h"
#include "lz4.h"
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+/**************************************
+ Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -178,9 +189,9 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define A16(x) (((U16_S *)(x))->v)
-//**************************************
-// Constants
-//**************************************
+/**************************************
+ Constants
+**************************************/
#define MINMATCH 4
#define DICTIONARY_LOGSIZE 16
@@ -192,8 +203,6 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
-#define MAX_NB_ATTEMPTS 256
-
#define ML_BITS 4
#define ML_MASK (size_t)((1U<<ML_BITS)-1)
#define RUN_BITS (8-ML_BITS)
@@ -210,25 +219,21 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#define GB *(1U<<30)
-//**************************************
-// Architecture-specific macros
-//**************************************
-#if LZ4_ARCH64 // 64-bit
+/**************************************
+ Architecture-specific macros
+**************************************/
+#if LZ4_ARCH64 /* 64-bit */
# define STEPSIZE 8
# define LZ4_COPYSTEP(s,d) A64(d) = A64(s); d+=8; s+=8;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d)
-# define UARCH U64
# define AARCH A64
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
-#else // 32-bit
+#else /* 32-bit */
# define STEPSIZE 4
# define LZ4_COPYSTEP(s,d) A32(d) = A32(s); d+=4; s+=4;
# define LZ4_COPYPACKET(s,d) LZ4_COPYSTEP(s,d); LZ4_COPYSTEP(s,d);
-# define UARCH U32
# define AARCH A32
-//# define HTYPE const BYTE*
-//# define INITBASE(b,s) const int b = 0
# define HTYPE U32
# define INITBASE(b,s) const BYTE* const b = s
#endif
@@ -236,16 +241,16 @@ typedef struct _U64_S { U64 v; } _PACKED U64_S;
#if defined(LZ4_BIG_ENDIAN)
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
+#else /* Little Endian */
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
#endif
-//************************************************************
-// Local Types
-//************************************************************
-typedef struct
+/**************************************
+ Local Types
+**************************************/
+typedef struct
{
const BYTE* inputBuffer;
const BYTE* base;
@@ -256,24 +261,24 @@ typedef struct
} LZ4HC_Data_Structure;
-//**************************************
-// Macros
-//**************************************
+/**************************************
+ Macros
+**************************************/
#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d<e);
#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=d+l; LZ4_WILDCOPY(s,d,e); d=e; }
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
#define HASH_VALUE(p) HASH_FUNCTION(A32(p))
#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base)
-#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
+#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK]
#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p))
-//**************************************
-// Private functions
-//**************************************
+/**************************************
+ Private functions
+**************************************/
#if LZ4_ARCH64
-forceinline int LZ4_NbCommonBytes (register U64 val)
+FORCE_INLINE int LZ4_NbCommonBytes (register U64 val)
{
#if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -281,7 +286,7 @@ forceinline int LZ4_NbCommonBytes (register U64 val)
_BitScanReverse64( &r, val );
return (int)(r>>3);
# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
+ return (__builtin_clzll(val) >> 3);
# else
int r;
if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
@@ -295,7 +300,7 @@ forceinline int LZ4_NbCommonBytes (register U64 val)
_BitScanForward64( &r, val );
return (int)(r>>3);
# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll(val) >> 3);
+ return (__builtin_ctzll(val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58];
@@ -305,7 +310,7 @@ forceinline int LZ4_NbCommonBytes (register U64 val)
#else
-forceinline int LZ4_NbCommonBytes (register U32 val)
+FORCE_INLINE int LZ4_NbCommonBytes (register U32 val)
{
#if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -313,7 +318,7 @@ forceinline int LZ4_NbCommonBytes (register U32 val)
_BitScanReverse( &r, val );
return (int)(r>>3);
# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz(val) >> 3);
+ return (__builtin_clz(val) >> 3);
# else
int r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
@@ -326,7 +331,7 @@ forceinline int LZ4_NbCommonBytes (register U32 val)
_BitScanForward( &r, val );
return (int)(r>>3);
# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz(val) >> 3);
+ return (__builtin_ctz(val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
@@ -337,7 +342,12 @@ forceinline int LZ4_NbCommonBytes (register U32 val)
#endif
-forceinline int LZ4_InitHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
+int LZ4_sizeofStreamStateHC()
+{
+ return sizeof(LZ4HC_Data_Structure);
+}
+
+FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
@@ -345,14 +355,20 @@ forceinline int LZ4_InitHC (LZ4HC_Data_Structure* hc4, const BYTE* base)
hc4->base = base;
hc4->inputBuffer = base;
hc4->end = base;
- return 1;
+}
+
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer)
+{
+ if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+ LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
+ return 0;
}
-void* LZ4_createHC (const char* slidingInputBuffer)
+void* LZ4_createHC (const char* inputBuffer)
{
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
- LZ4_InitHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)slidingInputBuffer);
+ LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
return hc4;
}
@@ -364,8 +380,8 @@ int LZ4_freeHC (void* LZ4HC_Data)
}
-// Update chains up to ip (excluded)
-forceinline void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
+/* Update chains up to ip (excluded) */
+FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
{
U16* chainTable = hc4->chainTable;
HTYPE* HashTable = hc4->hashTable;
@@ -374,9 +390,9 @@ forceinline void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
while(hc4->nextToUpdate < ip)
{
const BYTE* const p = hc4->nextToUpdate;
- size_t delta = (p) - HASH_POINTER(p);
- if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
- DELTANEXT(p) = (U16)delta;
+ size_t delta = (p) - HASH_POINTER(p);
+ if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
+ DELTANEXT(p) = (U16)delta;
HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base);
hc4->nextToUpdate++;
}
@@ -386,13 +402,16 @@ forceinline void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
- U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB;
- distance = (distance >> 16) << 16; // Must be a multiple of 64 KB
+ size_t distance = (hc4->end - 64 KB) - hc4->inputBuffer;
+
+ if (hc4->end <= hc4->inputBuffer + 64 KB) return (char*)(hc4->end); /* no update : less than 64KB within buffer */
+
+ distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */
LZ4HC_Insert(hc4, hc4->end - MINMATCH);
memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB);
hc4->nextToUpdate -= distance;
hc4->base -= distance;
- if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) // Avoid overflow
+ if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */
{
int i;
hc4->base += 1 GB;
@@ -403,13 +422,13 @@ char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
}
-forceinline size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
+FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit)
{
const BYTE* p1t = p1;
while (p1t<matchlimit-(STEPSIZE-1))
{
- UARCH diff = AARCH(p2) ^ AARCH(p1t);
+ size_t diff = AARCH(p2) ^ AARCH(p1t);
if (!diff) { p1t+=STEPSIZE; p2+=STEPSIZE; continue; }
p1t += LZ4_NbCommonBytes(diff);
return (p1t - p1);
@@ -421,26 +440,26 @@ forceinline size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYT
}
-forceinline int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos)
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos, const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
const BYTE* ref;
INITBASE(base,hc4->base);
- int nbAttempts=MAX_NB_ATTEMPTS;
+ int nbAttempts=maxNbAttempts;
size_t repl=0, ml=0;
- U16 delta=0; // useless assignment, to remove an uninitialization warning
+ U16 delta=0; /* useless assignment, to remove an uninitialization warning */
- // HC4 match finder
+ /* HC4 match finder */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
#define REPEAT_OPTIMIZATION
#ifdef REPEAT_OPTIMIZATION
- // Detect repetitive sequences of length <= 4
- if ((U32)(ip-ref) <= 4) // potential repetition
+ /* Detect repetitive sequences of length <= 4 */
+ if ((U32)(ip-ref) <= 4) /* potential repetition */
{
- if (A32(ref) == A32(ip)) // confirmed
+ if (A32(ref) == A32(ip)) /* confirmed */
{
delta = (U16)(ip-ref);
repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH;
@@ -463,7 +482,7 @@ forceinline int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const B
}
#ifdef REPEAT_OPTIMIZATION
- // Complete table
+ /* Complete table */
if (repl)
{
const BYTE* ptr = ip;
@@ -472,33 +491,33 @@ forceinline int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const B
end = ip + repl - (MINMATCH-1);
while(ptr < end-delta)
{
- DELTANEXT(ptr) = delta; // Pre-Load
+ DELTANEXT(ptr) = delta; /* Pre-Load */
ptr++;
}
do
{
- DELTANEXT(ptr) = delta;
- HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); // Head of chain
+ DELTANEXT(ptr) = delta;
+ HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */
ptr++;
} while(ptr < end);
hc4->nextToUpdate = end;
}
-#endif
+#endif
return (int)ml;
}
-forceinline int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos)
+FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
HTYPE* const HashTable = hc4->hashTable;
INITBASE(base,hc4->base);
const BYTE* ref;
- int nbAttempts = MAX_NB_ATTEMPTS;
+ int nbAttempts = maxNbAttempts;
int delta = (int)(ip-startLimit);
- // First Match
+ /* First Match */
LZ4HC_Insert(hc4, ip);
ref = HASH_POINTER(ip);
@@ -515,7 +534,7 @@ forceinline int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const B
while (ipt<matchlimit-(STEPSIZE-1))
{
- UARCH diff = AARCH(reft) ^ AARCH(ipt);
+ size_t diff = AARCH(reft) ^ AARCH(ipt);
if (!diff) { ipt+=STEPSIZE; reft+=STEPSIZE; continue; }
ipt += LZ4_NbCommonBytes(diff);
goto _endCount;
@@ -526,7 +545,7 @@ forceinline int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const B
_endCount:
reft = ref;
#else
- // Easier for code maintenance, but unfortunately slower too
+ /* Easier for code maintenance, but unfortunately slower too */
const BYTE* startt = ip;
const BYTE* reft = ref;
const BYTE* ipt = ip + MINMATCH + LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit);
@@ -548,37 +567,331 @@ _endCount:
}
+typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
-//**************************************
-// Compression functions
-//**************************************
+FORCE_INLINE int LZ4HC_encodeSequence (
+ const BYTE** ip,
+ BYTE** op,
+ const BYTE** anchor,
+ int matchLength,
+ const BYTE* ref,
+ limitedOutput_directive limitedOutputBuffer,
+ BYTE* oend)
+{
+ int length;
+ BYTE* token;
-/*
-int LZ4_compressHC(
- const char* source,
- char* dest,
- int inputSize)
+ /* Encode Literal length */
+ length = (int)(*ip - *anchor);
+ token = (*op)++;
+ if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */
+ if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
+ else *token = (BYTE)(length<<ML_BITS);
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest'.
-Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize).
-return : the number of bytes written in buffer 'dest'
-*/
-#define FUNCTION_NAME LZ4_compressHC
-#include "lz4hc_encoder.h"
+ /* Copy Literals */
+ LZ4_BLINDCOPY(*anchor, *op, length);
+ /* Encode Offset */
+ LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
-/*
-int LZ4_compressHC_limitedOutput(
+ /* Encode MatchLength */
+ length = (int)(matchLength-MINMATCH);
+ if ((limitedOutputBuffer) && (*op + (1 + LASTLITERALS) + (length>>8) > oend)) return 1; /* Check output limit */
+ if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
+ else *token += (BYTE)(length);
+
+ /* Prepare next loop */
+ *ip += matchLength;
+ *anchor = *ip;
+
+ return 0;
+}
+
+
+#define MAX_COMPRESSION_LEVEL 16
+static int LZ4HC_compress_generic (
+ void* ctxvoid,
const char* source,
char* dest,
int inputSize,
- int maxOutputSize)
+ int maxOutputSize,
+ int compressionLevel,
+ limitedOutput_directive limit
+ )
+{
+ LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* anchor = ip;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = (iend - LASTLITERALS);
+
+ BYTE* op = (BYTE*) dest;
+ BYTE* const oend = op + maxOutputSize;
+
+ const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<<LZ4HC_DEFAULT_COMPRESSIONLEVEL;
+ int ml, ml2, ml3, ml0;
+ const BYTE* ref=NULL;
+ const BYTE* start2=NULL;
+ const BYTE* ref2=NULL;
+ const BYTE* start3=NULL;
+ const BYTE* ref3=NULL;
+ const BYTE* start0;
+ const BYTE* ref0;
+
+
+ /* Ensure blocks follow each other */
+ if (ip != ctx->end) return 0;
+ ctx->end += inputSize;
+
+ ip++;
+
+ /* Main Loop */
+ while (ip < mflimit)
+ {
+ ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
+ if (!ml) { ip++; continue; }
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
-If it cannot achieve it, compression will stop, and result of the function will be zero.
-return : the number of bytes written in buffer 'dest', or 0 if the compression fails
-*/
-#define FUNCTION_NAME LZ4_compressHC_limitedOutput
-#define LIMITED_OUTPUT
-#include "lz4hc_encoder.h"
+ /* saved, in case we would skip too much */
+ start0 = ip;
+ ref0 = ref;
+ ml0 = ml;
+
+_Search2:
+ if (ip+ml < mflimit)
+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
+ else ml2 = ml;
+
+ if (ml2 == ml) /* No better match */
+ {
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ continue;
+ }
+
+ if (start0 < ip)
+ {
+ if (start2 < ip + ml0) /* empirical */
+ {
+ ip = start0;
+ ref = ref0;
+ ml = ml0;
+ }
+ }
+
+ /* Here, start0==ip */
+ if ((start2 - ip) < 3) /* First Match too small : removed */
+ {
+ ml = ml2;
+ ip = start2;
+ ref =ref2;
+ goto _Search2;
+ }
+
+_Search3:
+ /*
+ * Currently we have :
+ * ml2 > ml1, and
+ * ip1+3 <= ip2 (usually < ip1+ml1)
+ */
+ if ((start2 - ip) < OPTIMAL_ML)
+ {
+ int correction;
+ int new_ml = ml;
+ if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
+ if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
+ correction = new_ml - (int)(start2 - ip);
+ if (correction > 0)
+ {
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ }
+ }
+ /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
+
+ if (start2 + ml2 < mflimit)
+ ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
+ else ml3 = ml2;
+
+ if (ml3 == ml2) /* No better match : 2 sequences to encode */
+ {
+ /* ip & ref are known; Now for ml */
+ if (start2 < ip+ml) ml = (int)(start2 - ip);
+ /* Now, encode 2 sequences */
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ ip = start2;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
+ continue;
+ }
+
+ if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
+ {
+ if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
+ {
+ if (start2 < ip+ml)
+ {
+ int correction = (int)(ip+ml - start2);
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ if (ml2 < MINMATCH)
+ {
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+ }
+ }
+
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+ ip = start3;
+ ref = ref3;
+ ml = ml3;
+
+ start0 = start2;
+ ref0 = ref2;
+ ml0 = ml2;
+ goto _Search2;
+ }
+
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+ goto _Search3;
+ }
+ /*
+ * OK, now we have 3 ascending matches; let's write at least the first one
+ * ip & ref are known; Now for ml
+ */
+ if (start2 < ip+ml)
+ {
+ if ((start2 - ip) < (int)ML_MASK)
+ {
+ int correction;
+ if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
+ if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
+ correction = ml - (int)(start2 - ip);
+ if (correction > 0)
+ {
+ start2 += correction;
+ ref2 += correction;
+ ml2 -= correction;
+ }
+ }
+ else
+ {
+ ml = (int)(start2 - ip);
+ }
+ }
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+
+ ip = start2;
+ ref = ref2;
+ ml = ml2;
+
+ start2 = start3;
+ ref2 = ref3;
+ ml2 = ml3;
+
+ goto _Search3;
+
+ }
+
+ /* Encode Last Literals */
+ {
+ int lastRun = (int)(iend - anchor);
+ if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
+ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+ else *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend-anchor;
+ }
+
+ /* End */
+ return (int) (((char*)op)-dest);
+}
+
+
+int LZ4_compressHC2(const char* source, char* dest, int inputSize, int compressionLevel)
+{
+ void* ctx = LZ4_createHC(source);
+ int result;
+ if (ctx==NULL) return 0;
+
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, 0, compressionLevel, noLimit);
+
+ LZ4_freeHC(ctx);
+ return result;
+}
+
+int LZ4_compressHC(const char* source, char* dest, int inputSize) { return LZ4_compressHC2(source, dest, inputSize, 0); }
+
+int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+ void* ctx = LZ4_createHC(source);
+ int result;
+ if (ctx==NULL) return 0;
+
+ result = LZ4HC_compress_generic (ctx, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+
+ LZ4_freeHC(ctx);
+ return result;
+}
+
+int LZ4_compressHC_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_compressHC2_limitedOutput(source, dest, inputSize, maxOutputSize, 0);
+}
+
+
+/*****************************
+ Using external allocation
+*****************************/
+int LZ4_sizeofStateHC() { return sizeof(LZ4HC_Data_Structure); }
+
+
+int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel)
+{
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
+ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, 0, compressionLevel, noLimit);
+}
+
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize)
+{ return LZ4_compressHC2_withStateHC (state, source, dest, inputSize, 0); }
+
+
+int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+ if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
+ LZ4_initHC ((LZ4HC_Data_Structure*)state, (const BYTE*)source);
+ return LZ4HC_compress_generic (state, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+}
+
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{ return LZ4_compressHC2_limitedOutput_withStateHC (state, source, dest, inputSize, maxOutputSize, 0); }
+
+
+/****************************
+ Stream functions
+****************************/
+
+int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, 0, noLimit);
+}
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
+}
+
+int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, 0, limitedOutput);
+}
+
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+ return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h
index 2bdbdb0..deb2394 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h
@@ -1,7 +1,7 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -44,7 +44,7 @@ int LZ4_compressHC (const char* source, char* dest, int inputSize);
LZ4_compressHC :
return : the number of bytes in compressed buffer dest
or 0 if compression fails.
- note : destination buffer must be already allocated.
+ note : destination buffer must be already allocated.
To avoid any problem, size it to handle worst cases situations (input data not compressible)
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
*/
@@ -63,38 +63,76 @@ LZ4_compress_limitedOutput() :
*/
+int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+/*
+ Same functions as above, but with programmable 'compressionLevel'.
+ Recommended values are between 4 and 9, although any value between 0 and 16 will work.
+ 'compressionLevel'==0 means use default 'compressionLevel' value.
+ Values above 16 behave the same as 16.
+ Equivalent variants exist for all other compression functions below.
+*/
+
/* Note :
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
-/* Advanced Functions */
+/**************************************
+ Using an external allocation
+**************************************/
+int LZ4_sizeofStateHC(void);
+int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
+These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
+To know how much memory must be allocated for the compression tables, use :
+int LZ4_sizeofStateHC();
-void* LZ4_createHC (const char* slidingInputBuffer);
+Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
+
+The allocated memory can be provided to the compressions functions using 'void* state' parameter.
+LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
+They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
+*/
+
+
+/**************************************
+ Streaming Functions
+**************************************/
+/* Note : these streaming functions still follows the older model */
+void* LZ4_createHC (const char* inputBuffer);
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
int LZ4_freeHC (void* LZ4HC_Data);
-/*
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+
+/*
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
-void* LZ4_createHC (const char* slidingInputBuffer);
+void* LZ4_createHC (const char* inputBuffer);
The result of the function is the (void*) pointer on the LZ4HC Data Structure.
This pointer will be needed in all other functions.
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
-The only parameter 'const char* slidingInputBuffer' must, obviously, point at the beginning of input buffer.
+The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
The input buffer must be already allocated, and size at least 192KB.
-'slidingInputBuffer' will also be the 'const char* source' of the first block.
+'inputBuffer' will also be the 'const char* source' of the first block.
-All blocks are expected to lay next to each other within the input buffer, starting from 'slidingInputBuffer'.
+All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
-Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
+Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
If next block does not begin immediately after the previous one, the compression will fail (return 0).
-When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
+When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
@@ -105,6 +143,30 @@ Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_l
When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
*/
+int LZ4_sizeofStreamStateHC(void);
+int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+
+/*
+These functions achieve the same result as :
+void* LZ4_createHC (const char* inputBuffer);
+
+They are provided here to allow the user program to allocate memory using its own routines.
+
+To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
+Note also that space must be aligned for pointers (32 or 64 bits).
+
+Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
+void* state is a pointer to the space allocated.
+It must be aligned for pointers (32 or 64 bits), and be large enough.
+The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
+The input buffer must be already allocated, and size at least 192KB.
+'inputBuffer' will also be the 'const char* source' of the first block.
+
+The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
+return value of LZ4_resetStreamStateHC() must be 0 is OK.
+Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
+*/
+
#if defined (__cplusplus)
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc_encoder.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc_encoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc_encoder.h
deleted file mode 100644
index 6c53743..0000000
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc_encoder.h
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- LZ4 HC Encoder - Part of LZ4 HC algorithm
- Copyright (C) 2011-2013, Yann Collet.
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- - LZ4 source repository : http://code.google.com/p/lz4/
-*/
-
-/* lz4hc_encoder.h must be included into lz4hc.c
- The objective of this file is to create a single LZ4 compression function source
- which will be instanciated multiple times with minor variations
- depending on a set of #define.
-*/
-
-//****************************
-// Check required defines
-//****************************
-
-#ifndef FUNCTION_NAME
-# error "FUNTION_NAME is not defined"
-#endif
-
-
-//****************************
-// Local definitions
-//****************************
-#define COMBINED_NAME_RAW(n1,n2) n1 ## n2
-#define COMBINED_NAME(n1,n2) COMBINED_NAME_RAW(n1,n2)
-#define ENCODE_SEQUENCE_NAME COMBINED_NAME(FUNCTION_NAME,_encodeSequence)
-#ifdef LIMITED_OUTPUT
-# define ENCODE_SEQUENCE(i,o,a,m,r,d) if (ENCODE_SEQUENCE_NAME(i,o,a,m,r,d)) return 0;
-#else
-# define ENCODE_SEQUENCE(i,o,a,m,r,d) ENCODE_SEQUENCE_NAME(i,o,a,m,r)
-#endif
-
-//****************************
-// Function code
-//****************************
-
-forceinline int ENCODE_SEQUENCE_NAME (
- const BYTE** ip,
- BYTE** op,
- const BYTE** anchor,
- int matchLength,
- const BYTE* ref
-#ifdef LIMITED_OUTPUT
- ,BYTE* oend
-#endif
- )
-{
- int length, len;
- BYTE* token;
-
- // Encode Literal length
- length = (int)(*ip - *anchor);
- token = (*op)++;
-#ifdef LIMITED_OUTPUT
- if ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend) return 1; // Check output limit
-#endif
- if (length>=(int)RUN_MASK) { *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
- else *token = (BYTE)(length<<ML_BITS);
-
- // Copy Literals
- LZ4_BLINDCOPY(*anchor, *op, length);
-
- // Encode Offset
- LZ4_WRITE_LITTLEENDIAN_16(*op,(U16)(*ip-ref));
-
- // Encode MatchLength
- length = (int)(matchLength-MINMATCH);
-#ifdef LIMITED_OUTPUT
- if (*op + (1 + LASTLITERALS) + (length>>8) > oend) return 1; // Check output limit
-#endif
- if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
- else *token += (BYTE)(length);
-
- // Prepare next loop
- *ip += matchLength;
- *anchor = *ip;
-
- return 0;
-}
-
-
-int COMBINED_NAME(FUNCTION_NAME,_continue) (
- void* ctxvoid,
- const char* source,
- char* dest,
- int inputSize
-#ifdef LIMITED_OUTPUT
- ,int maxOutputSize
-#endif
- )
-{
- LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
- const BYTE* ip = (const BYTE*) source;
- const BYTE* anchor = ip;
- const BYTE* const iend = ip + inputSize;
- const BYTE* const mflimit = iend - MFLIMIT;
- const BYTE* const matchlimit = (iend - LASTLITERALS);
-
- BYTE* op = (BYTE*) dest;
-#ifdef LIMITED_OUTPUT
- BYTE* const oend = op + maxOutputSize;
-#endif
-
- int ml, ml2, ml3, ml0;
- const BYTE* ref=NULL;
- const BYTE* start2=NULL;
- const BYTE* ref2=NULL;
- const BYTE* start3=NULL;
- const BYTE* ref3=NULL;
- const BYTE* start0;
- const BYTE* ref0;
-
- // Ensure blocks follow each other
- if (ip != ctx->end) return 0;
- ctx->end += inputSize;
-
- ip++;
-
- // Main Loop
- while (ip < mflimit)
- {
- ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));
- if (!ml) { ip++; continue; }
-
- // saved, in case we would skip too much
- start0 = ip;
- ref0 = ref;
- ml0 = ml;
-
-_Search2:
- if (ip+ml < mflimit)
- ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
- else ml2 = ml;
-
- if (ml2 == ml) // No better match
- {
- ENCODE_SEQUENCE(&ip, &op, &anchor, ml, ref, oend);
- continue;
- }
-
- if (start0 < ip)
- {
- if (start2 < ip + ml0) // empirical
- {
- ip = start0;
- ref = ref0;
- ml = ml0;
- }
- }
-
- // Here, start0==ip
- if ((start2 - ip) < 3) // First Match too small : removed
- {
- ml = ml2;
- ip = start2;
- ref =ref2;
- goto _Search2;
- }
-
-_Search3:
- // Currently we have :
- // ml2 > ml1, and
- // ip1+3 <= ip2 (usually < ip1+ml1)
- if ((start2 - ip) < OPTIMAL_ML)
- {
- int correction;
- int new_ml = ml;
- if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
- if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
- correction = new_ml - (int)(start2 - ip);
- if (correction > 0)
- {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- }
- // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18)
-
- if (start2 + ml2 < mflimit)
- ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
- else ml3 = ml2;
-
- if (ml3 == ml2) // No better match : 2 sequences to encode
- {
- // ip & ref are known; Now for ml
- if (start2 < ip+ml) ml = (int)(start2 - ip);
- // Now, encode 2 sequences
- ENCODE_SEQUENCE(&ip, &op, &anchor, ml, ref, oend);
- ip = start2;
- ENCODE_SEQUENCE(&ip, &op, &anchor, ml2, ref2, oend);
- continue;
- }
-
- if (start3 < ip+ml+3) // Not enough space for match 2 : remove it
- {
- if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
- {
- if (start2 < ip+ml)
- {
- int correction = (int)(ip+ml - start2);
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- if (ml2 < MINMATCH)
- {
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- }
- }
-
- ENCODE_SEQUENCE(&ip, &op, &anchor, ml, ref, oend);
- ip = start3;
- ref = ref3;
- ml = ml3;
-
- start0 = start2;
- ref0 = ref2;
- ml0 = ml2;
- goto _Search2;
- }
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- goto _Search3;
- }
-
- // OK, now we have 3 ascending matches; let's write at least the first one
- // ip & ref are known; Now for ml
- if (start2 < ip+ml)
- {
- if ((start2 - ip) < (int)ML_MASK)
- {
- int correction;
- if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
- if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
- correction = ml - (int)(start2 - ip);
- if (correction > 0)
- {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- }
- else
- {
- ml = (int)(start2 - ip);
- }
- }
- ENCODE_SEQUENCE(&ip, &op, &anchor, ml, ref, oend);
-
- ip = start2;
- ref = ref2;
- ml = ml2;
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
-
- goto _Search3;
-
- }
-
- // Encode Last Literals
- {
- int lastRun = (int)(iend - anchor);
-#ifdef LIMITED_OUTPUT
- if (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) return 0; // Check output limit
-#endif
- if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
- else *op++ = (BYTE)(lastRun<<ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend-anchor;
- }
-
- // End
- return (int) (((char*)op)-dest);
-}
-
-
-int FUNCTION_NAME (const char* source,
- char* dest,
- int inputSize
-#ifdef LIMITED_OUTPUT
- ,int maxOutputSize
-#endif
- )
-{
- void* ctx = LZ4_createHC(source);
- int result;
- if (ctx==NULL) return 0;
-#ifdef LIMITED_OUTPUT
- result = COMBINED_NAME(FUNCTION_NAME,_continue) (ctx, source, dest, inputSize, maxOutputSize);
-#else
- result = COMBINED_NAME(FUNCTION_NAME,_continue) (ctx, source, dest, inputSize);
-#endif
- LZ4_freeHC(ctx);
-
- return result;
-}
-
-
-//****************************
-// Clean defines
-//****************************
-
-// Required defines
-#undef FUNCTION_NAME
-
-// Locally Generated
-#undef ENCODE_SEQUENCE
-#undef ENCODE_SEQUENCE_NAME
-
-// Optional defines
-#ifdef LIMITED_OUTPUT
-#undef LIMITED_OUTPUT
-#endif
-
-
[24/50] [abbrv] git commit: HDFS-7179. DFSClient should instantiate a
KeyProvider, not a KeyProviderCryptoExtension. (wang)
Posted by st...@apache.org.
HDFS-7179. DFSClient should instantiate a KeyProvider, not a KeyProviderCryptoExtension. (wang)
(cherry picked from commit d2d5a0ea03b0d461a4d376c7b9de8cd5c147effa)
(cherry picked from commit 6ddd9eff78a15081dfe3b405b3d0a958ff27acf5)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ebbc995
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ebbc995
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ebbc995
Branch: refs/heads/champlain
Commit: 3ebbc995b427ba796a4272541790888399d49e53
Parents: 09ac019
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 2 13:50:05 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:36:57 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 11 +++++----
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 25 ++++++++++++++++----
.../apache/hadoop/hdfs/TestEncryptionZones.java | 3 +--
4 files changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebbc995/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c70bd9e..cf39e25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -538,6 +538,12 @@ Release 2.6.0 - UNRELEASED
HDFS-6754. TestNamenodeCapacityReport may sometimes fail due to lack of
retry. (Mit Desai via kihwal)
+ HDFS-7178. Additional unit test for replica write with full disk.
+ (Arpit Agarwal)
+
+ HDFS-7179. DFSClient should instantiate a KeyProvider, not a
+ KeyProviderCryptoExtension. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebbc995/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index d83d8cb..c975ad5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.crypto.CryptoCodec;
import org.apache.hadoop.crypto.CryptoInputStream;
import org.apache.hadoop.crypto.CryptoOutputStream;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStorageLocation;
@@ -264,7 +265,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
new DFSHedgedReadMetrics();
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
@VisibleForTesting
- KeyProviderCryptoExtension provider;
+ KeyProvider provider;
/**
* DFSClient configuration
*/
@@ -596,7 +597,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
- provider = DFSUtil.createKeyProviderCryptoExtension(conf);
+ provider = DFSUtil.createKeyProvider(conf);
if (LOG.isDebugEnabled()) {
if (provider == null) {
LOG.debug("No KeyProvider found.");
@@ -1315,7 +1316,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
feInfo.getKeyName(), feInfo.getEzKeyVersionName(), feInfo.getIV(),
feInfo.getEncryptedDataEncryptionKey());
try {
- return provider.decryptEncryptedKey(ekv);
+ KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
+ .createKeyProviderCryptoExtension(provider);
+ return cryptoProvider.decryptEncryptedKey(ekv);
} catch (GeneralSecurityException e) {
throw new IOException(e);
}
@@ -3138,7 +3141,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return HEDGED_READ_METRIC;
}
- public KeyProviderCryptoExtension getKeyProvider() {
+ public KeyProvider getKeyProvider() {
return provider;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebbc995/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index c76da2c..151028f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -1825,15 +1825,14 @@ public class DFSUtil {
}
/**
- * Creates a new KeyProviderCryptoExtension by wrapping the
- * KeyProvider specified in the given Configuration.
+ * Creates a new KeyProvider from the given Configuration.
*
* @param conf Configuration
- * @return new KeyProviderCryptoExtension, or null if no provider was found.
+ * @return new KeyProvider, or null if no provider was found.
* @throws IOException if the KeyProvider is improperly specified in
* the Configuration
*/
- public static KeyProviderCryptoExtension createKeyProviderCryptoExtension(
+ public static KeyProvider createKeyProvider(
final Configuration conf) throws IOException {
final String providerUriStr =
conf.get(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, null);
@@ -1857,6 +1856,24 @@ public class DFSUtil {
throw new IOException("KeyProvider " + keyProvider.toString()
+ " was found but it is a transient provider.");
}
+ return keyProvider;
+ }
+
+ /**
+ * Creates a new KeyProviderCryptoExtension by wrapping the
+ * KeyProvider specified in the given Configuration.
+ *
+ * @param conf Configuration
+ * @return new KeyProviderCryptoExtension, or null if no provider was found.
+ * @throws IOException if the KeyProvider is improperly specified in
+ * the Configuration
+ */
+ public static KeyProviderCryptoExtension createKeyProviderCryptoExtension(
+ final Configuration conf) throws IOException {
+ KeyProvider keyProvider = createKeyProvider(conf);
+ if (keyProvider == null) {
+ return null;
+ }
KeyProviderCryptoExtension cryptoProvider = KeyProviderCryptoExtension
.createKeyProviderCryptoExtension(keyProvider);
return cryptoProvider;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebbc995/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index df1864c..c384bfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
@@ -1043,7 +1042,7 @@ public class TestEncryptionZones {
public void testDelegationToken() throws Exception {
UserGroupInformation.createRemoteUser("JobTracker");
DistributedFileSystem dfs = cluster.getFileSystem();
- KeyProviderCryptoExtension keyProvider = Mockito.mock(KeyProviderCryptoExtension.class,
+ KeyProvider keyProvider = Mockito.mock(KeyProvider.class,
withSettings().extraInterfaces(
DelegationTokenExtension.class,
CryptoExtension.class));
[06/50] [abbrv] git commit: HDFS-7156. Update fsck documentation.
Contributed by Masahiro Yamaguch.
Posted by st...@apache.org.
HDFS-7156. Update fsck documentation. Contributed by Masahiro Yamaguch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39132478
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39132478
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39132478
Branch: refs/heads/champlain
Commit: 391324780ad9af8e19eec42b2459012e1d5aff57
Parents: ae1634f
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Sat Sep 27 21:24:22 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:49:56 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../src/site/apt/HDFSCommands.apt.vm | 51 ++++++++++++--------
2 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39132478/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0a37e7a..0a0f50e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -278,6 +278,8 @@ Release 2.6.0 - UNRELEASED
HDFS-6956. Allow dynamically changing the tracing level in Hadoop servers
(cmccabe)
+ HDFS-7156. Update fsck documentation. (Masahiro Yamaguch via shv)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39132478/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 76ea90d..1e73377 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -82,28 +82,37 @@ HDFS Commands Guide
See {{{./HdfsUserGuide.html#fsck}fsck}} for more info.
Usage: <<<hdfs fsck [GENERIC_OPTIONS] <path>
+ [-list-corruptfileblocks |
[-move | -delete | -openforwrite]
- [-files [-blocks [-locations | -racks]]]>>>
-
-*------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------+---------------------------------------------+
-| <path> | Start checking from this path.
-*------------------+---------------------------------------------+
-| -move | Move corrupted files to /lost+found
-*------------------+---------------------------------------------+
-| -delete | Delete corrupted files.
-*------------------+---------------------------------------------+
-| -openforwrite | Print out files opened for write.
-*------------------+---------------------------------------------+
-| -files | Print out files being checked.
-*------------------+---------------------------------------------+
-| -blocks | Print out block report.
-*------------------+---------------------------------------------+
-| -locations | Print out locations for every block.
-*------------------+---------------------------------------------+
-| -racks | Print out network topology for data-node locations.
-*------------------+---------------------------------------------+
+ [-files [-blocks [-locations | -racks]]]]
+ [-includeSnapshots]>>>
+
+*------------------------+---------------------------------------------+
+|| COMMAND_OPTION || Description
+*------------------------+---------------------------------------------+
+| <path> | Start checking from this path.
+*------------------------+---------------------------------------------+
+| -move | Move corrupted files to /lost+found
+*------------------------+---------------------------------------------+
+| -delete | Delete corrupted files.
+*------------------------+---------------------------------------------+
+| -files | Print out files being checked.
+*------------------------+---------------------------------------------+
+| -openforwrite | Print out files opened for write.
+*------------------------+---------------------------------------------+
+| | Include snapshot data if the given path
+| -includeSnapshots | indicates a snapshottable directory or
+| | there are snapshottable directories under it.
+*------------------------+---------------------------------------------+
+| -list-corruptfileblocks| Print out list of missing blocks and
+| | files they belong to.
+*------------------------+---------------------------------------------+
+| -blocks | Print out block report.
+*------------------------+---------------------------------------------+
+| -locations | Print out locations for every block.
+*------------------------+---------------------------------------------+
+| -racks | Print out network topology for data-node locations.
+*------------------------+---------------------------------------------+
* Administration Commands
[14/50] [abbrv] git commit: HDFS-7124. Remove
EncryptionZoneManager.NULL_EZ. (clamb via wang)
Posted by st...@apache.org.
HDFS-7124. Remove EncryptionZoneManager.NULL_EZ. (clamb via wang)
(cherry picked from commit 7f0efe96f85704e39349f20ab6a11bfaa81c6a75)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cb02592
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cb02592
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cb02592
Branch: refs/heads/champlain
Commit: 6cb025924a569066a93454b88e55b722d3c6f052
Parents: 2ce35f8
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 29 14:14:22 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:13:26 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 3 +--
.../ClientNamenodeProtocolServerSideTranslatorPB.java | 4 +++-
.../hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java | 6 +++++-
.../hadoop/hdfs/server/namenode/EncryptionZoneManager.java | 6 +-----
.../org/apache/hadoop/hdfs/server/namenode/FSDirectory.java | 3 +--
.../hadoop-hdfs/src/main/proto/encryption.proto | 2 +-
7 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 25bfbb7..506620f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -289,6 +289,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7104. Fix and clarify INodeInPath getter functions. (Zhe Zhang via wang)
+ HDFS-7124. Remove EncryptionZoneManager.NULL_EZ. (clamb via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index bdfea12..d83d8cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2940,8 +2940,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
throws IOException {
checkOpen();
try {
- final EncryptionZone ez = namenode.getEZForPath(src);
- return (ez.getId() < 0) ? null : ez;
+ return namenode.getEZForPath(src);
} catch (RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
UnresolvedPathException.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 27904e1..a92d455 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -1340,7 +1340,9 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetEZForPathResponseProto.Builder builder =
GetEZForPathResponseProto.newBuilder();
final EncryptionZone ret = server.getEZForPath(req.getSrc());
- builder.setZone(PBHelper.convert(ret));
+ if (ret != null) {
+ builder.setZone(PBHelper.convert(ret));
+ }
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index df8965c..077a3e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -1347,7 +1347,11 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
final EncryptionZonesProtos.GetEZForPathResponseProto response =
rpcProxy.getEZForPath(null, req);
- return PBHelper.convert(response.getZone());
+ if (response.hasZone()) {
+ return PBHelper.convert(response.getZone());
+ } else {
+ return null;
+ }
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 0e83583..0d7ced9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -57,10 +57,6 @@ public class EncryptionZoneManager {
public static Logger LOG = LoggerFactory.getLogger(EncryptionZoneManager
.class);
- public static final EncryptionZone NULL_EZ =
- new EncryptionZone(-1, "", CipherSuite.UNKNOWN,
- CryptoProtocolVersion.UNKNOWN, "");
-
/**
* EncryptionZoneInt is the internal representation of an encryption zone. The
* external representation of an EZ is embodied in an EncryptionZone and
@@ -226,7 +222,7 @@ public class EncryptionZoneManager {
EncryptionZone getEZINodeForPath(INodesInPath iip) {
final EncryptionZoneInt ezi = getEncryptionZoneForPath(iip);
if (ezi == null) {
- return NULL_EZ;
+ return null;
} else {
return new EncryptionZone(ezi.getINodeId(), getFullPathName(ezi),
ezi.getSuite(), ezi.getVersion(), ezi.getKeyName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c817b88..d57bc82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -2869,8 +2869,7 @@ public class FSDirectory implements Closeable {
iip = getINodesInPath(inode.getFullPathName(), true);
}
EncryptionZone encryptionZone = getEZForPath(iip);
- if (encryptionZone == null ||
- encryptionZone.equals(EncryptionZoneManager.NULL_EZ)) {
+ if (encryptionZone == null) {
// not an encrypted file
return null;
} else if(encryptionZone.getPath() == null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb02592/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
index bb291ea..68b2f3a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto
@@ -63,5 +63,5 @@ message GetEZForPathRequestProto {
}
message GetEZForPathResponseProto {
- required EncryptionZoneProto zone = 1;
+ optional EncryptionZoneProto zone = 1;
}
[35/50] [abbrv] git commit: BUG-24553 hadoop champlain build failure:
registry linking to ZK
Posted by st...@apache.org.
BUG-24553 hadoop champlain build failure: registry linking to ZK
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89019847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89019847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89019847
Branch: refs/heads/champlain
Commit: 89019847b8ac048e218c705d0f7873b2394b458e
Parents: 5e5503b
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 9 09:46:14 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 9 09:46:14 2014 -0700
----------------------------------------------------------------------
.../hadoop/registry/client/impl/zk/RegistrySecurity.java | 6 +++---
.../hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89019847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 6484d28..6d5792e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.util.ZKUtil;
import org.apache.zookeeper.Environment;
import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
@@ -758,7 +757,7 @@ public class RegistrySecurity extends AbstractService {
* <b>Important:</b>This is JVM-wide
*/
public static void disableZookeeperClientSASL() {
- System.setProperty(ZooKeeperSaslClient.ENABLE_CLIENT_SASL_KEY, "false");
+ System.setProperty(ZookeeperConfigOptions.PROP_ZK_ENABLE_SASL_CLIENT, "false");
}
/**
@@ -766,7 +765,8 @@ public class RegistrySecurity extends AbstractService {
* @return true if the SASL client system property is set.
*/
public static boolean isClientSASLEnabled() {
- return ZooKeeperSaslClient.isEnabled();
+ return Boolean.valueOf(System.getProperty(
+ ZookeeperConfigOptions.PROP_ZK_ENABLE_SASL_CLIENT, "true"));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89019847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java
index 711e27c..d81f24b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ZookeeperConfigOptions.java
@@ -47,7 +47,7 @@ public interface ZookeeperConfigOptions {
* but revert to classic authentication/anonymous access on failure.
*/
String PROP_ZK_ENABLE_SASL_CLIENT =
- ZooKeeperSaslClient.ENABLE_CLIENT_SASL_KEY;
+ "zookeeper.sasl.client";
/**
* Default flag for the ZK client: {@value}.
[49/50] [abbrv] git commit: Merge branch 'champlain' of
https://github.com/hortonworks/hadoop-monarch into champlain
Posted by st...@apache.org.
Merge branch 'champlain' of https://github.com/hortonworks/hadoop-monarch into champlain
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b774d29a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b774d29a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b774d29a
Branch: refs/heads/champlain
Commit: b774d29ad213d5b547b93d3ce98328c24dc31760
Parents: b7f34c7 5d29917
Author: Eric Hanson <ehans at microsoft dot com>
Authored: Fri Oct 10 10:56:45 2014 -0700
Committer: Eric Hanson <ehans at microsoft dot com>
Committed: Fri Oct 10 10:56:45 2014 -0700
----------------------------------------------------------------------
.gitignore | 2 +
BUILDING.txt | 2 +-
LICENSE.txt | 7 +-
.../authentication/client/AuthenticatedURL.java | 5 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 79 ++
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-common-project/hadoop-common/pom.xml | 2 +-
.../hadoop-common/src/main/bin/hadoop | 3 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 11 +-
.../apache/hadoop/conf/ReconfigurableBase.java | 140 ++
.../hadoop/conf/ReconfigurationTaskStatus.java | 70 +
.../apache/hadoop/conf/ReconfigurationUtil.java | 5 +
.../org/apache/hadoop/crypto/CipherSuite.java | 10 -
.../org/apache/hadoop/crypto/CryptoCodec.java | 7 +-
.../hadoop/crypto/CryptoProtocolVersion.java | 90 ++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 58 +-
.../apache/hadoop/crypto/key/KeyProvider.java | 8 +
.../crypto/key/KeyProviderCryptoExtension.java | 7 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 6 +-
.../crypto/key/kms/KMSClientProvider.java | 63 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 14 +-
.../hadoop/fs/CommonConfigurationKeys.java | 3 +
.../java/org/apache/hadoop/fs/FileContext.java | 3 +-
.../apache/hadoop/fs/FileEncryptionInfo.java | 30 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 6 +-
.../io/compress/snappy/SnappyCompressor.java | 24 +-
.../io/compress/snappy/SnappyDecompressor.java | 26 +-
.../hadoop/io/compress/zlib/ZlibCompressor.java | 24 +-
.../io/compress/zlib/ZlibDecompressor.java | 30 +-
.../java/org/apache/hadoop/net/NetUtils.java | 2 +-
.../org/apache/hadoop/net/NetworkTopology.java | 25 +-
.../hadoop/net/unix/DomainSocketWatcher.java | 17 +-
.../java/org/apache/hadoop/security/User.java | 3 +-
.../hadoop/security/UserGroupInformation.java | 18 +-
.../web/DelegationTokenAuthenticatedURL.java | 12 +-
.../web/DelegationTokenAuthenticator.java | 4 +-
.../apache/hadoop/tracing/SpanReceiverHost.java | 136 +-
.../apache/hadoop/tracing/SpanReceiverInfo.java | 64 +
.../hadoop/tracing/SpanReceiverInfoBuilder.java | 42 +
.../org/apache/hadoop/tracing/TraceAdmin.java | 197 +++
.../hadoop/tracing/TraceAdminProtocol.java | 69 +
.../hadoop/tracing/TraceAdminProtocolPB.java | 38 +
...raceAdminProtocolServerSideTranslatorPB.java | 118 ++
.../tracing/TraceAdminProtocolTranslatorPB.java | 105 ++
.../hadoop/util/ApplicationClassLoader.java | 47 +-
.../org/apache/hadoop/util/ProgramDriver.java | 1 -
.../java/org/apache/hadoop/util/RunJar.java | 4 +-
.../src/org/apache/hadoop/io/compress/lz4/lz4.c | 1243 +++++++++++++-----
.../src/org/apache/hadoop/io/compress/lz4/lz4.h | 296 +++--
.../apache/hadoop/io/compress/lz4/lz4_encoder.h | 258 ----
.../org/apache/hadoop/io/compress/lz4/lz4hc.c | 591 +++++++--
.../org/apache/hadoop/io/compress/lz4/lz4hc.h | 84 +-
.../hadoop/io/compress/lz4/lz4hc_encoder.h | 349 -----
.../src/main/proto/TraceAdmin.proto | 73 +
.../src/main/resources/core-default.xml | 7 +
...he.hadoop.application-classloader.properties | 57 +
.../src/site/apt/FileSystemShell.apt.vm | 55 +-
.../src/site/apt/SecureMode.apt.vm | 70 +-
.../apache/hadoop/conf/TestReconfiguration.java | 157 ++-
.../FailureInjectingJavaKeyStoreProvider.java | 80 ++
.../crypto/key/TestKeyProviderFactory.java | 48 +-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 6 +-
.../hadoop/io/compress/TestCodecPool.java | 87 ++
.../security/TestUserGroupInformation.java | 3 +-
.../java/org/apache/hadoop/util/TestRunJar.java | 2 +-
....apache.hadoop.crypto.key.KeyProviderFactory | 19 +
.../hadoop-kms/src/main/conf/kms-env.sh | 4 +
.../hadoop-kms/src/main/libexec/kms-config.sh | 7 +
.../hadoop-kms/src/main/sbin/kms.sh | 1 +
.../hadoop-kms/src/main/tomcat/server.xml | 1 +
.../hadoop-kms/src/main/tomcat/ssl-server.xml | 2 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 2 +
.../hadoop/crypto/key/kms/server/MiniKMS.java | 20 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 230 +++-
.../org/apache/hadoop/nfs/nfs3/IdUserGroup.java | 15 +-
hadoop-dist/src/win/scripts/InstallApi.psm1 | 2 +-
hadoop-dist/src/win/scripts/install.ps1 | 23 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 72 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
.../hadoop-hdfs/src/CMakeLists.txt | 17 +-
.../hadoop-hdfs/src/main/bin/hdfs | 5 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 138 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 14 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 25 +-
.../apache/hadoop/hdfs/HDFSPolicyProvider.java | 6 +-
.../hdfs/UnknownCipherSuiteException.java | 11 +-
.../UnknownCryptoProtocolVersionException.java | 38 +
.../hdfs/protocol/ClientDatanodeProtocol.java | 12 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 5 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 47 +-
...tDatanodeProtocolServerSideTranslatorPB.java | 61 +-
.../ClientDatanodeProtocolTranslatorPB.java | 55 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 9 +-
.../ClientNamenodeProtocolTranslatorPB.java | 13 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 119 +-
.../server/blockmanagement/BlockManager.java | 13 +-
.../blockmanagement/DatanodeDescriptor.java | 15 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 80 +-
.../server/namenode/EncryptionZoneManager.java | 62 +-
.../hdfs/server/namenode/FSDirectory.java | 147 ++-
.../hdfs/server/namenode/FSEditLogLoader.java | 8 +-
.../hdfs/server/namenode/FSNamesystem.java | 126 +-
.../hdfs/server/namenode/INodesInPath.java | 24 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 8 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 41 +-
.../hadoop/hdfs/server/namenode/StreamFile.java | 16 +-
.../hdfs/server/protocol/NamenodeProtocols.java | 4 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 95 +-
.../src/main/native/fuse-dfs/fuse_trash.c | 7 +-
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 2 +-
.../main/native/libhdfs/test/test_libhdfs_ops.c | 22 +-
.../src/main/native/libhdfs/test/vecsum.c | 33 +-
.../main/native/libhdfs/test_libhdfs_threaded.c | 2 +-
.../src/main/proto/ClientDatanodeProtocol.proto | 30 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +-
.../hadoop-hdfs/src/main/proto/encryption.proto | 10 +-
.../hadoop-hdfs/src/main/proto/hdfs.proto | 40 +-
.../src/main/resources/hdfs-default.xml | 49 +
.../src/site/apt/HDFSCommands.apt.vm | 57 +-
.../src/site/apt/HdfsImageViewer.apt.vm | 59 +-
.../src/site/apt/HdfsPermissionsGuide.apt.vm | 13 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 3 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 142 +-
.../org/apache/hadoop/hdfs/TestFileAppend3.java | 83 ++
.../java/org/apache/hadoop/hdfs/TestLease.java | 3 +-
.../TestUnderReplicatedBlocks.java | 77 ++
.../namenode/TestNamenodeCapacityReport.java | 4 +-
.../server/namenode/TestSnapshotPathINodes.java | 6 +-
.../hdfs/server/namenode/TestStreamFile.java | 7 +-
.../namenode/ha/TestRetryCacheWithHA.java | 4 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 152 +++
.../apache/hadoop/tracing/TestTraceAdmin.java | 90 ++
hadoop-mapreduce-project/CHANGES.txt | 6 +
.../hadoop/mapreduce/v2/util/TestMRApps.java | 4 +-
.../hadoop-mapreduce-client-core/.gitattributes | 1 +
.../hadoop-mapreduce-client-core/pom.xml | 1 +
.../lib/input/TestCombineFileInputFormat.java | 4 -
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +-
hadoop-project/pom.xml | 5 +
.../hadoop/tools/CopyListingFileStatus.java | 2 +-
.../apache/hadoop/tools/DistCpOptionSwitch.java | 21 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 2 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 58 +-
.../apache/hadoop/tools/TestOptionsParser.java | 2 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 1 +
.../hadoop/tools/util/TestDistCpUtils.java | 1024 +++++++++++++--
.../src/site/markdown/GridMix.md.vm | 818 ++++++++++++
.../hadoop-rumen/src/site/markdown/Rumen.md.vm | 397 ++++++
hadoop-yarn-project/CHANGES.txt | 15 +
.../dev-support/findbugs-exclude.xml | 10 +
.../distributedshell/ApplicationMaster.java | 120 +-
.../applications/distributedshell/Client.java | 75 +-
.../distributedshell/DSConstants.java | 5 +
.../distributedshell/TestDistributedShell.java | 52 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 31 +-
.../client/api/impl/TimelineClientImpl.java | 3 -
.../logaggregation/LogAggregationUtils.java | 5 +
.../yarn/logaggregation/LogCLIHelpers.java | 105 +-
.../yarn/webapp/log/AggregatedLogsBlock.java | 207 +--
.../src/main/resources/yarn-default.xml | 126 ++
.../logaggregation/TestAggregatedLogsBlock.java | 3 -
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 218 +++
.../apache/hadoop/registry/cli/RegistryCli.java | 445 +++++++
.../hadoop/registry/client/api/BindFlags.java | 41 +
.../registry/client/api/RegistryConstants.java | 286 ++++
.../registry/client/api/RegistryOperations.java | 182 +++
.../client/api/RegistryOperationsFactory.java | 131 ++
.../registry/client/api/package-info.java | 35 +
.../registry/client/binding/JsonSerDeser.java | 327 +++++
.../client/binding/RegistryPathUtils.java | 218 +++
.../client/binding/RegistryTypeUtils.java | 240 ++++
.../registry/client/binding/RegistryUtils.java | 362 +++++
.../registry/client/binding/package-info.java | 22 +
.../AuthenticationFailedException.java | 39 +
.../exceptions/InvalidPathnameException.java | 40 +
.../exceptions/InvalidRecordException.java | 41 +
.../NoChildrenForEphemeralsException.java | 48 +
.../exceptions/NoPathPermissionsException.java | 45 +
.../client/exceptions/NoRecordException.java | 51 +
.../client/exceptions/RegistryIOException.java | 58 +
.../client/exceptions/package-info.java | 33 +
.../client/impl/RegistryOperationsClient.java | 55 +
.../registry/client/impl/package-info.java | 26 +
.../client/impl/zk/BindingInformation.java | 41 +
.../registry/client/impl/zk/CuratorService.java | 769 +++++++++++
.../client/impl/zk/RegistryBindingSource.java | 36 +
.../impl/zk/RegistryInternalConstants.java | 81 ++
.../impl/zk/RegistryOperationsService.java | 155 +++
.../client/impl/zk/RegistrySecurity.java | 996 ++++++++++++++
.../registry/client/impl/zk/ZKPathDumper.java | 133 ++
.../client/impl/zk/ZookeeperConfigOptions.java | 119 ++
.../registry/client/impl/zk/package-info.java | 39 +
.../registry/client/types/AddressTypes.java | 92 ++
.../hadoop/registry/client/types/Endpoint.java | 190 +++
.../registry/client/types/ProtocolTypes.java | 104 ++
.../client/types/RegistryPathStatus.java | 123 ++
.../registry/client/types/ServiceRecord.java | 249 ++++
.../client/types/ServiceRecordHeader.java | 59 +
.../registry/client/types/package-info.java | 41 +
.../client/types/yarn/PersistencePolicies.java | 50 +
.../types/yarn/YarnRegistryAttributes.java | 31 +
.../RMRegistryOperationsService.java | 246 ++++
.../integration/SelectByYarnPersistence.java | 60 +
.../server/integration/package-info.java | 23 +
.../hadoop/registry/server/package-info.java | 27 +
.../server/services/AddingCompositeService.java | 56 +
.../services/DeleteCompletionCallback.java | 58 +
.../server/services/MicroZookeeperService.java | 282 ++++
.../services/MicroZookeeperServiceKeys.java | 69 +
.../server/services/RegistryAdminService.java | 529 ++++++++
.../registry/server/services/package-info.java | 40 +
.../src/main/resources/.keep | 0
.../src/main/tla/yarnregistry.tla | 538 ++++++++
.../hadoop/registry/AbstractRegistryTest.java | 123 ++
.../hadoop/registry/AbstractZKRegistryTest.java | 113 ++
.../hadoop/registry/RegistryTestHelper.java | 401 ++++++
.../client/binding/TestMarshalling.java | 121 ++
.../binding/TestRegistryOperationUtils.java | 47 +
.../client/binding/TestRegistryPathUtils.java | 178 +++
.../client/impl/CuratorEventCatcher.java | 68 +
.../client/impl/TestCuratorService.java | 249 ++++
.../client/impl/TestMicroZookeeperService.java | 60 +
.../integration/TestRegistryRMOperations.java | 369 ++++++
.../integration/TestYarnPolicySelector.java | 65 +
.../operations/TestRegistryOperations.java | 304 +++++
.../secure/AbstractSecureRegistryTest.java | 356 +++++
.../registry/secure/KerberosConfiguration.java | 81 ++
.../secure/TestRegistrySecurityHelper.java | 211 +++
.../registry/secure/TestSecureLogins.java | 214 +++
.../secure/TestSecureRMRegistryOperations.java | 350 +++++
.../registry/secure/TestSecureRegistry.java | 157 +++
.../src/test/resources/log4j.properties | 63 +
...pplicationHistoryManagerOnTimelineStore.java | 20 +
...pplicationHistoryManagerOnTimelineStore.java | 23 +-
.../util/TestCgroupsLCEResourcesHandler.java | 7 +-
.../hadoop-yarn-site/src/site/apt/index.apt.vm | 2 +
.../src/site/markdown/registry/index.md | 28 +
.../site/markdown/registry/registry-security.md | 120 ++
.../registry/using-the-yarn-service-registry.md | 150 +++
.../src/site/markdown/registry/yarn-registry.md | 889 +++++++++++++
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 +
241 files changed, 21519 insertions(+), 2152 deletions(-)
----------------------------------------------------------------------
[34/50] [abbrv] git commit: HADOOP-11161. Expose close method in
KeyProvider to give clients of Provider implementations a hook to release
resources. Contribued by Arun Suresh. (cherry picked from commit
d9556e873ef4d3e68c4f0c991f856d1faa747f07) (cherry
Posted by st...@apache.org.
HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations a hook to release resources. Contribued by Arun Suresh.
(cherry picked from commit d9556e873ef4d3e68c4f0c991f856d1faa747f07)
(cherry picked from commit 3a2565c7be80cf6e9cdfec0f5460ed8ed2252768)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e5503b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e5503b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e5503b8
Branch: refs/heads/champlain
Commit: 5e5503b8d93b8413ba2268d7058575b1665a7db4
Parents: 3cbe727
Author: Aaron T. Myers <at...@apache.org>
Authored: Wed Oct 8 17:58:53 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:30:12 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/KeyProvider.java | 8 ++++++++
.../crypto/key/KeyProviderCryptoExtension.java | 7 +++++++
.../hadoop/crypto/key/kms/KMSClientProvider.java | 11 +++++++++++
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 14 +++++++++++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++------
6 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index db2795c..69b7622 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -469,6 +469,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
by lock (cmccabe)
+ HADOOP-11161. Expose close method in KeyProvider to give clients of
+ Provider implementations a hook to release resources. (Arun Suresh via atm)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a8b9414..9dd1d47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -534,6 +534,14 @@ public abstract class KeyProvider {
) throws IOException;
/**
+ * Can be used by implementing classes to close any resources
+ * that require closing
+ */
+ public void close() throws IOException {
+ // NOP
+ }
+
+ /**
* Roll a new version of the given key generating the material for it.
* <p/>
* This implementation generates the key material and calls the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index f800689..73c9885 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -410,4 +410,11 @@ public class KeyProviderCryptoExtension extends
return new KeyProviderCryptoExtension(keyProvider, cryptoExtension);
}
+ @Override
+ public void close() throws IOException {
+ if (getKeyProvider() != null) {
+ getKeyProvider().close();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 5b7f109..c4c7e0c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -791,4 +791,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return tokens;
}
+ /**
+ * Shutdown valueQueue executor threads
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ encKeyVersionQueue.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index ee10483..aa0e624 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -75,6 +75,8 @@ public class ValueQueue <E> {
private final int numValues;
private final float lowWatermark;
+ private volatile boolean executorThreadsStarted = false;
+
/**
* A <code>Runnable</code> which takes a string name.
*/
@@ -187,9 +189,6 @@ public class ValueQueue <E> {
TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat(REFILL_THREAD).build());
- // To ensure all requests are first queued, make coreThreads = maxThreads
- // and pre-start all the Core Threads.
- executor.prestartAllCoreThreads();
}
public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
@@ -297,6 +296,15 @@ public class ValueQueue <E> {
private void submitRefillTask(final String keyName,
final Queue<E> keyQueue) throws InterruptedException {
+ if (!executorThreadsStarted) {
+ synchronized (this) {
+ // To ensure all requests are first queued, make coreThreads =
+ // maxThreads
+ // and pre-start all the Core Threads.
+ executor.prestartAllCoreThreads();
+ executorThreadsStarted = true;
+ }
+ }
// The submit/execute method of the ThreadPoolExecutor is bypassed and
// the Runnable is directly put in the backing BlockingQueue so that we
// can control exactly how the runnable is inserted into the queue.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 66a4d0c..c26e3d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -920,12 +920,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*/
@Override
public synchronized void close() throws IOException {
- if(clientRunning) {
- closeAllFilesBeingWritten(false);
- clientRunning = false;
- getLeaseRenewer().closeClient(this);
- // close connections to the namenode
- closeConnectionToNamenode();
+ try {
+ if(clientRunning) {
+ closeAllFilesBeingWritten(false);
+ clientRunning = false;
+ getLeaseRenewer().closeClient(this);
+ // close connections to the namenode
+ closeConnectionToNamenode();
+ }
+ } finally {
+ if (provider != null) {
+ provider.close();
+ }
}
}
[30/50] [abbrv] git commit: HADOOP-11168. Remove duplicated entry
dfs.webhdfs.enabled in the user doc. Contributed by Yi Liu.
Posted by st...@apache.org.
HADOOP-11168. Remove duplicated entry dfs.webhdfs.enabled in the user doc. Contributed by Yi Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e1e11ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e1e11ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e1e11ea
Branch: refs/heads/champlain
Commit: 7e1e11ea41f4f5fedeaea58083059bff30f7758f
Parents: 23cd97a
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Oct 6 22:07:16 2014 -0600
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:14:54 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/apt/SecureMode.apt.vm | 3 ---
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e1e11ea/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 04cc594..afa18bc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -460,6 +460,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11151. Automatically refresh auth token and retry on auth failure.
(Arun Suresh via wang)
+ HADOOP-11168. Remove duplicated entry "dfs.webhdfs.enabled" in the useri
+ doc. (Yi Liu via wheat9)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e1e11ea/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
index c02ea39..f42db2d 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
@@ -469,9 +469,6 @@ Configuration for <<<conf/hdfs-site.xml>>>
*-------------------------+-------------------------+------------------------+
|| Parameter || Value || Notes |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.webhdfs.enabled>>> | http/_HOST@REALM.TLD | |
-| | | Enable security on WebHDFS. |
-*-------------------------+-------------------------+------------------------+
| <<<dfs.web.authentication.kerberos.principal>>> | http/_HOST@REALM.TLD | |
| | | Kerberos keytab file for the WebHDFS. |
*-------------------------+-------------------------+------------------------+
[07/50] [abbrv] git commit: HADOOP-11049. javax package system class
default is too broad. Contributed by Sangjin Lee (cherry picked from commit
cf4631d7c9448205f4952341c071e1650e7922dc)
Posted by st...@apache.org.
HADOOP-11049. javax package system class default is too broad. Contributed by Sangjin Lee
(cherry picked from commit cf4631d7c9448205f4952341c071e1650e7922dc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/215e1325
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/215e1325
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/215e1325
Branch: refs/heads/champlain
Commit: 215e13252b91b8f103694adf489b59fda435b623
Parents: 3913247
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Sep 29 14:36:29 2014 +0000
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:54:51 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/util/ApplicationClassLoader.java | 47 +++++++++++-----
...he.hadoop.application-classloader.properties | 57 ++++++++++++++++++++
.../java/org/apache/hadoop/util/TestRunJar.java | 2 +-
.../hadoop/mapreduce/v2/util/TestMRApps.java | 4 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +-
6 files changed, 98 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 922bae8..68a145d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -570,6 +570,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11143 NetUtils.wrapException loses inner stack trace on BindException
(stevel)
+ HADOOP-11049. javax package system class default is too broad (Sangjin Lee
+ via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index 5dda10f..b18997c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -45,18 +48,12 @@ public class ApplicationClassLoader extends URLClassLoader {
* classes are considered system classes, and are not loaded by the
* application classloader.
*/
- public static final String DEFAULT_SYSTEM_CLASSES =
- "java.," +
- "javax.," +
- "org.w3c.dom.," +
- "org.xml.sax.," +
- "org.apache.commons.logging.," +
- "org.apache.log4j.," +
- "org.apache.hadoop.," +
- "core-default.xml," +
- "hdfs-default.xml," +
- "mapred-default.xml," +
- "yarn-default.xml";
+ public static final String SYSTEM_CLASSES_DEFAULT;
+
+ private static final String PROPERTIES_FILE =
+ "org.apache.hadoop.application-classloader.properties";
+ private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+ "system.classes.default";
private static final Log LOG =
LogFactory.getLog(ApplicationClassLoader.class.getName());
@@ -69,6 +66,30 @@ public class ApplicationClassLoader extends URLClassLoader {
}
};
+ static {
+ InputStream is = null;
+ try {
+ is = ApplicationClassLoader.class.getClassLoader().
+ getResourceAsStream(PROPERTIES_FILE);
+ if (is == null) {
+ throw new ExceptionInInitializerError("properties file " +
+ PROPERTIES_FILE + " is not found");
+ }
+ Properties props = new Properties();
+ props.load(is);
+ // get the system classes default
+ String systemClassesDefault =
+ props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+ if (systemClassesDefault == null) {
+ throw new ExceptionInInitializerError("property " +
+ SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
+ }
+ SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
+ } catch (IOException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+
private final ClassLoader parent;
private final List<String> systemClasses;
@@ -85,7 +106,7 @@ public class ApplicationClassLoader extends URLClassLoader {
}
// if the caller-specified system classes are null or empty, use the default
this.systemClasses = (systemClasses == null || systemClasses.isEmpty()) ?
- Arrays.asList(StringUtils.getTrimmedStrings(DEFAULT_SYSTEM_CLASSES)) :
+ Arrays.asList(StringUtils.getTrimmedStrings(SYSTEM_CLASSES_DEFAULT)) :
systemClasses;
LOG.info("system classes: " + this.systemClasses);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties b/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties
new file mode 100644
index 0000000..2264920
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/org.apache.hadoop.application-classloader.properties
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# contains key properties for setting up the application classloader
+system.classes.default=java.,\
+ javax.accessibility.,\
+ javax.activation.,\
+ javax.activity.,\
+ javax.annotation.,\
+ javax.annotation.processing.,\
+ javax.crypto.,\
+ javax.imageio.,\
+ javax.jws.,\
+ javax.lang.model.,\
+ -javax.management.j2ee.,\
+ javax.management.,\
+ javax.naming.,\
+ javax.net.,\
+ javax.print.,\
+ javax.rmi.,\
+ javax.script.,\
+ -javax.security.auth.message.,\
+ javax.security.auth.,\
+ javax.security.cert.,\
+ javax.security.sasl.,\
+ javax.sound.,\
+ javax.sql.,\
+ javax.swing.,\
+ javax.tools.,\
+ javax.transaction.,\
+ -javax.xml.registry.,\
+ -javax.xml.rpc.,\
+ javax.xml.,\
+ org.w3c.dom.,\
+ org.xml.sax.,\
+ org.apache.commons.logging.,\
+ org.apache.log4j.,\
+ org.apache.hadoop.,\
+ core-default.xml,\
+ hdfs-default.xml,\
+ mapred-default.xml,\
+ yarn-default.xml
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 9e27968..f592d04 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -131,7 +131,7 @@ public class TestRunJar extends TestCase {
String thirdCls = ClassLoaderCheckThird.class.getName();
String systemClasses = "-" + mainCls + "," +
"-" + thirdCls + "," +
- ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES;
+ ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT;
when(runJar.getSystemClasses()).thenReturn(systemClasses);
// create the test jar
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 3b20bf1..2e77f5e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -494,7 +494,7 @@ public class TestMRApps {
private static final String[] SYS_CLASSES = new String[] {
"/java/fake/Klass",
- "/javax/fake/Klass",
+ "/javax/management/fake/Klass",
"/org/apache/commons/logging/fake/Klass",
"/org/apache/log4j/fake/Klass",
"/org/apache/hadoop/fake/Klass"
@@ -511,7 +511,7 @@ public class TestMRApps {
public void testSystemClasses() {
final List<String> systemClasses =
Arrays.asList(StringUtils.getTrimmedStrings(
- ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES));
+ ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT));
for (String defaultXml : DEFAULT_XMLS) {
assertTrue(defaultXml + " must be system resource",
ApplicationClassLoader.isSystemClass(defaultXml, systemClasses));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/215e1325/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 5699600..e27168d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -242,7 +242,7 @@ public class TestMRJobs {
// to test AM loading user classes such as output format class, we want
// to blacklist them from the system classes (they need to be prepended
// as the first match wins)
- String systemClasses = ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES;
+ String systemClasses = ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT;
// exclude the custom classes from system classes
systemClasses = "-" + CustomOutputFormat.class.getName() + ",-" +
CustomSpeculator.class.getName() + "," +
[43/50] [abbrv] git commit: MAPREDUCE-6122. TestLineRecordReader may
fail due to test data files checked out of git with incorrect line endings.
Contributed by Chris Nauroth.
Posted by st...@apache.org.
MAPREDUCE-6122. TestLineRecordReader may fail due to test data files checked out of git with incorrect line endings. Contributed by Chris Nauroth.
(cherry picked from commit 684170d5efdc11bcd9967168f44a428d1a72af3c)
(cherry picked from commit 85a2b907407c7719a31c64d3747810263b3ec57d)
(cherry picked from commit 3fe4e0d3dfa90e9001d79cb37cc879bab0e0136f)
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45e6bd54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45e6bd54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45e6bd54
Branch: refs/heads/champlain
Commit: 45e6bd5417e302be1112448753b36318ca041088
Parents: 97fa1fc
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 22:25:17 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 22:54:29 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop-mapreduce-client-core/.gitattributes | 1 +
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 1 +
3 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45e6bd54/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 32b8f15..cbc71b2 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
(Akira AJISAKA via jlowe)
+ MAPREDUCE-6029. TestLineRecordReader may fail due to test data files checked
+ out of git with incorrect line endings. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45e6bd54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/.gitattributes
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/.gitattributes b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/.gitattributes
new file mode 100644
index 0000000..cbee8c4
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/.gitattributes
@@ -0,0 +1 @@
+src/test/resources/*.txt text eol=lf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45e6bd54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index f77d896..321d64c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -90,6 +90,7 @@
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>.gitattributes</exclude>
<exclude>src/test/resources/recordSpanningMultipleSplits.txt</exclude>
<exclude>src/test/resources/testBOM.txt</exclude>
</excludes>
[46/50] [abbrv] git commit: YARN-2662. TestCgroupsLCEResourcesHandler
leaks file descriptors. Contributed by Chris Nauroth.
Posted by st...@apache.org.
YARN-2662. TestCgroupsLCEResourcesHandler leaks file descriptors. Contributed by Chris Nauroth.
(cherry picked from commit d3afd730acfa380ab5032be5ee296c5d73744518)
(cherry picked from commit b81641a31050b6a5cf5db3af636125e0617ffd56)
(cherry picked from commit 281041a57e56708a0ec6767b44d88f3765c57c94)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/616f3094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/616f3094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/616f3094
Branch: refs/heads/champlain
Commit: 616f3094d1d49ee5dde8d7c9b478257817cd456c
Parents: 5794c70
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 22:44:44 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 22:56:11 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../nodemanager/util/TestCgroupsLCEResourcesHandler.java | 7 ++++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/616f3094/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 07c0514..8cbcb27 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -503,6 +503,8 @@ Release 2.6.0 - UNRELEASED
YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
reservation even though slots are free. (Varun Vasudev via jianhe)
+ YARN-2662. TestCgroupsLCEResourcesHandler leaks file descriptors. (cnauroth)
+
BREAKDOWN OF YARN-913 SUBTASKS AND RELATED JIRAS
YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/616f3094/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index de2af37..d0bceee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -165,10 +165,11 @@ public class TestCgroupsLCEResourcesHandler {
private int readIntFromFile(File targetFile) throws IOException {
Scanner scanner = new Scanner(targetFile);
- if (scanner.hasNextInt()) {
- return scanner.nextInt();
+ try {
+ return scanner.hasNextInt() ? scanner.nextInt() : -1;
+ } finally {
+ scanner.close();
}
- return -1;
}
@Test
[47/50] [abbrv] git commit: BUG-13755 MSI installer should support
installing, configuring, starting Oozie server on multiple nodes
Posted by st...@apache.org.
BUG-13755 MSI installer should support installing, configuring, starting Oozie server on multiple nodes
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d29917e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d29917e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d29917e
Branch: refs/heads/champlain
Commit: 5d29917e993a9498a243ce6f4f0908c752b58c9c
Parents: 616f309
Author: ivanmalamen <ma...@ukr.net>
Authored: Fri Oct 10 12:58:31 2014 +0300
Committer: ivanmalamen <ma...@ukr.net>
Committed: Fri Oct 10 12:58:31 2014 +0300
----------------------------------------------------------------------
hadoop-dist/src/win/scripts/InstallApi.psm1 | 2 +-
hadoop-dist/src/win/scripts/install.ps1 | 18 +++++++++++-------
2 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d29917e/hadoop-dist/src/win/scripts/InstallApi.psm1
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/win/scripts/InstallApi.psm1 b/hadoop-dist/src/win/scripts/InstallApi.psm1
index 1dd5613..b36720c 100644
--- a/hadoop-dist/src/win/scripts/InstallApi.psm1
+++ b/hadoop-dist/src/win/scripts/InstallApi.psm1
@@ -741,7 +741,7 @@ function UninstallHdfs(
### Stop and delete services
###
$services = @("namenode", "datanode", "secondarynamenode")
- if ((Test-Path ENV:HA) -and ($ENV:HA -eq "yes"))
+ if ((Test-Path ENV:IS_HDFS_HA) -and ($ENV:IS_HDFS_HA -eq "yes"))
{
$services+=@("journalnode","zkfc")
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d29917e/hadoop-dist/src/win/scripts/install.ps1
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/win/scripts/install.ps1 b/hadoop-dist/src/win/scripts/install.ps1
index d9c54e0..425fdfb 100644
--- a/hadoop-dist/src/win/scripts/install.ps1
+++ b/hadoop-dist/src/win/scripts/install.ps1
@@ -144,7 +144,7 @@ function Main( $scriptDir )
}
if (($role -eq "NAMENODE") -or ($role -eq "NN_HA_STANDBY_NAMENODE")) {
$hdfsroles = $hdfsroles+" "+"namenode"
- if ($ENV:HA -ieq "yes") {
+ if ($ENV:IS_HDFS_HA -ieq "yes") {
$hdfsroles = $hdfsroles+" "+"zkfc"
}
}
@@ -154,7 +154,7 @@ function Main( $scriptDir )
if ($role -eq "SECONDARYNAMENODE" ) {
$hdfsroles = $hdfsroles+" "+"secondarynamenode"
}
- if (($role -eq "RM_HA_STANDBY_RESOURCEMANAGER" ) -and ($ENV:HA -ieq "yes")) {
+ if (($role -eq "RM_HA_STANDBY_RESOURCEMANAGER" ) -and ($ENV:IS_HDFS_HA -ieq "yes")) {
$hdfsroles = $hdfsroles+" "+"zkfc"
$yarnRoles = $yarnRoles+" "+"resourcemanager"
$yarnRoles = $yarnRoles+" "+"timelineserver"
@@ -195,7 +195,11 @@ function Main( $scriptDir )
### Prepare the proxyuser hosts
$proxyHostsList = @()
$proxyHostsList += GetIPAddress $ENV:HIVE_SERVER_HOST
- $proxyHostsList += GetIPAddress $ENV:OOZIE_SERVER_HOST
+ $oozie_hosts = ($ENV:OOZIE_SERVER_HOST.Split(",") | foreach { $_.Trim() })
+ foreach ($shost in $oozie_hosts)
+ {
+ $proxyHostsList += GetIPAddress $shost
+ }
$proxyHostsList += GetIPAddress $ENV:WEBHCAT_HOST
$proxyHostsList = $proxyHostsList | Select -Uniq
$proxyHosts = ($proxyHostsList -Join ',')
@@ -233,7 +237,7 @@ function Main( $scriptDir )
Write-Log "HDFS usage detected. Configuring HDP to use HDFS as default filesystem"
$coreConfigs["fs.defaultFS"] = "hdfs://${ENV:NAMENODE_HOST}:8020"
- if ($ENV:HA -ieq "yes") {
+ if ($ENV:IS_HDFS_HA -ieq "yes") {
$coreConfigs["fs.defaultFS"] = "hdfs://${ENV:NN_HA_CLUSTER_NAME}"
$zookeeperNodes = $zookeeperNodes = $env:ZOOKEEPER_HOSTS.Replace(",",":2181,")
$zookeeperNodes = $zookeeperNodes + ":2181"
@@ -308,7 +312,7 @@ function Main( $scriptDir )
"dfs.hosts" = "${hadoopInstallDir}\etc\hadoop\dfs.include";
"dfs.hosts.exclude" = "${hadoopInstallDir}\etc\hadoop\dfs.exclude";
"dfs.support.append" = "true"}
- if ($ENV:HA -ieq "yes") {
+ if ($ENV:IS_HDFS_HA -ieq "yes") {
$hdfsConfigs["dfs.nameservices"] = "${ENV:NN_HA_CLUSTER_NAME}"
$hdfsConfigs["dfs.ha.namenodes.${ENV:NN_HA_CLUSTER_NAME}"] = "nn1,nn2"
$hdfsConfigs["dfs.namenode.rpc-address.${ENV:NN_HA_CLUSTER_NAME}.nn1"] = "${ENV:NAMENODE_HOST}:8020"
@@ -388,7 +392,7 @@ function Main( $scriptDir )
"yarn.nodemanager.log-dirs" = "$NMAndMRLogDir" ;
"yarn.nodemanager.local-dirs" = "$NMAndMRLocalDir" ;
"yarn.timeline-service.hostname" = "${ENV:RESOURCEMANAGER_HOST}".ToLower() }
- if ($ENV:HA -ieq "yes") {
+ if ($ENV:IS_HDFS_HA -ieq "yes") {
$yarnConfigs += @{
"yarn.resourcemanager.ha.enabled" = "true";
"yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
@@ -427,7 +431,7 @@ function Main( $scriptDir )
"mapreduce.map.java.opts" = "-Xmx756m";
"mapreduce.cluster.local.dir" = "$NMAndMRLocalDir" }
- if ((Test-Path ENV:HA) -and ($ENV:HA -ieq "yes")) {
+ if ((Test-Path ENV:IS_HDFS_HA) -and ($ENV:IS_HDFS_HA -ieq "yes")) {
$mapredConfigs["mapreduce.am.max-attempts"] = "20"
}
if ((Test-Path ENV:ENABLE_GZIP) -and ($ENV:ENABLE_GZIP -ieq "yes")) {
[42/50] [abbrv] git commit: HADOOP-11184. Update Hadoop's lz4 to r123
(cmccabe) (cherry picked from commit
d8d628d1e0c42a32e2079ed430e8817996bbe4c4) (cherry picked from commit
8addbe2b8eaf9fabbb0b6dc0dd3351a0145e0cae)
Posted by st...@apache.org.
HADOOP-11184. Update Hadoop's lz4 to r123 (cmccabe)
(cherry picked from commit d8d628d1e0c42a32e2079ed430e8817996bbe4c4)
(cherry picked from commit 8addbe2b8eaf9fabbb0b6dc0dd3351a0145e0cae)
(cherry picked from commit 14080ea3d22d6fc637b8c7f8f0152256bbd5ee4a)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97fa1fc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97fa1fc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97fa1fc3
Branch: refs/heads/champlain
Commit: 97fa1fc36f7ed322694bcc73d2a291626e3bcd0a
Parents: b45cbc8
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Oct 9 17:05:35 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 21:48:57 2014 -0700
----------------------------------------------------------------------
LICENSE.txt | 7 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-common/pom.xml | 1 -
.../src/org/apache/hadoop/io/compress/lz4/lz4.c | 1243 +++++++++++++-----
.../src/org/apache/hadoop/io/compress/lz4/lz4.h | 296 +++--
.../apache/hadoop/io/compress/lz4/lz4_encoder.h | 258 ----
.../org/apache/hadoop/io/compress/lz4/lz4hc.c | 591 +++++++--
.../org/apache/hadoop/io/compress/lz4/lz4hc.h | 84 +-
.../hadoop/io/compress/lz4/lz4hc_encoder.h | 349 -----
9 files changed, 1657 insertions(+), 1174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 946a6df..8a4b175 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -252,13 +252,12 @@ in src/main/native/src/org/apache/hadoop/util:
* BSD-style license that can be found in the LICENSE file.
*/
-For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,
-lz4_encoder.h,lz4hc.h,lz4hc.c,lz4hc_encoder.h},
+For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c},
/*
LZ4 - Fast LZ compression algorithm
Header File
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -285,6 +284,6 @@ lz4_encoder.h,lz4hc.h,lz4hc.c,lz4hc_encoder.h},
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7f41469..30f5f92 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -232,6 +232,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11153. Make number of KMS threads configurable. (wang)
+ HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index cc8db1f..02be9e5 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -505,7 +505,6 @@
<exclude>src/test/resources/kdc/ldif/users.ldif</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude>
- <exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4_encoder.h</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.h</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc.c</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4hc_encoder.h</exclude>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c
index 8eda7ae..39f176f 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -27,47 +27,44 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
+/**************************************
+ Tuning parameters
+**************************************/
/*
-Note : this source file requires "lz4_encoder.h"
-*/
-
-//**************************************
-// Tuning parameters
-//**************************************
-// MEMORY_USAGE :
-// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
-// Increasing memory usage improves compression ratio
-// Reduced memory usage can improve speed, due to cache effect
-// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
-#define MEMORY_USAGE 14
-
-// HEAPMODE :
-// Select how default compression function will allocate memory for its hash table,
-// in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
-// Default allocation strategy is to use stack (HEAPMODE 0)
-// Note : explicit functions *_stack* and *_heap* are unaffected by this setting
+ * HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires memory allocation (malloc)).
+ */
#define HEAPMODE 0
-//**************************************
-// CPU Feature Detection
-//**************************************
-// 32 or 64 bits ?
+/**************************************
+ CPU Feature Detection
+**************************************/
+/* 32 or 64 bits ? */
#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \
- || defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) \
- || defined(__64BIT__) || defined(_LP64) || defined(__LP64__) \
- || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) // Detects 64 bits mode
+ || defined(__64BIT__) || defined(__mips64) \
+ || defined(__powerpc64__) || defined(__powerpc64le__) \
+ || defined(__ppc64__) || defined(__ppc64le__) \
+ || defined(__PPC64__) || defined(__PPC64LE__) \
+ || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) \
+ || defined(__s390x__) ) /* Detects 64 bits mode */
# define LZ4_ARCH64 1
#else
# define LZ4_ARCH64 0
#endif
+#define LZ4_32BITS (sizeof(void*)==4)
+#define LZ4_64BITS (sizeof(void*)==8)
-// Little Endian or Big Endian ?
-// Overwrite the #define below if you know your architecture endianess
+/*
+ * Little Endian or Big Endian ?
+ * Overwrite the #define below if you know your architecture endianess
+ */
+#include <stdlib.h> /* Apparently required to detect endianess */
#if defined (__GLIBC__)
# include <endian.h>
# if (__BYTE_ORDER == __BIG_ENDIAN)
@@ -81,57 +78,62 @@ Note : this source file requires "lz4_encoder.h"
|| defined(_MIPSEB) || defined(__s390__)
# define LZ4_BIG_ENDIAN 1
#else
-// Little Endian assumed. PDP Endian and other very rare endian format are unsupported.
+/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */
#endif
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+/*
+ * Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, such as ARM, the compiler may be more cautious, inserting unnecessary extra code to ensure aligned access property
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance
+ */
#if defined(__ARM_FEATURE_UNALIGNED)
# define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
-// Define this parameter if your target system or compiler does not support hardware bit count
-#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count
+/* Define this parameter if your target system or compiler does not support hardware bit count */
+#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LZ4_FORCE_SW_BITCOUNT
#endif
-// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
-// This option may provide a small boost to performance for some big endian cpu, although probably modest.
-// You may set this option to 1 if data will remain within closed environment.
-// This option is useless on Little_Endian CPU (such as x86)
-//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1
+/*
+ * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE :
+ * This option may provide a small boost to performance for some big endian cpu, although probably modest.
+ * You may set this option to 1 if data will remain within closed environment.
+ * This option is useless on Little_Endian CPU (such as x86)
+ */
+
+/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) // C99
+/**************************************
+ Compiler Options
+**************************************/
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
/* "restrict" is a known keyword */
#else
-# define restrict // Disable restrict
+# define restrict /* Disable restrict */
#endif
-#ifdef _MSC_VER // Visual Studio
-# define forceinline static __forceinline
-# include <intrin.h> // For Visual 2005
-# if LZ4_ARCH64 // 64-bits
-# pragma intrinsic(_BitScanForward64) // For Visual 2005
-# pragma intrinsic(_BitScanReverse64) // For Visual 2005
-# else // 32-bits
-# pragma intrinsic(_BitScanForward) // For Visual 2005
-# pragma intrinsic(_BitScanReverse) // For Visual 2005
+#ifdef _MSC_VER /* Visual Studio */
+# define FORCE_INLINE static __forceinline
+# include <intrin.h> /* For Visual 2005 */
+# if LZ4_ARCH64 /* 64-bits */
+# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */
+# else /* 32-bits */
+# pragma intrinsic(_BitScanForward) /* For Visual 2005 */
+# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */
# endif
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
-#else
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+#else
# ifdef __GNUC__
-# define forceinline static inline __attribute__((always_inline))
+# define FORCE_INLINE static inline __attribute__((always_inline))
# else
-# define forceinline static inline
+# define FORCE_INLINE static inline
# endif
#endif
-#ifdef _MSC_VER
+#ifdef _MSC_VER /* Visual Studio */
# define lz4_bswap16(x) _byteswap_ushort(x)
#else
# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8)))
@@ -149,18 +151,26 @@ Note : this source file requires "lz4_encoder.h"
#define unlikely(expr) expect((expr) != 0, 0)
-//**************************************
-// Includes
-//**************************************
-#include <stdlib.h> // for malloc
-#include <string.h> // for memset
+/**************************************
+ Memory routines
+**************************************/
+#include <stdlib.h> /* malloc, calloc, free */
+#define ALLOCATOR(n,s) calloc(n,s)
+#define FREEMEM free
+#include <string.h> /* memset, memcpy */
+#define MEM_INIT memset
+
+
+/**************************************
+ Includes
+**************************************/
#include "lz4.h"
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
+/**************************************
+ Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -182,7 +192,7 @@ Note : this source file requires "lz4_encoder.h"
#endif
#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# ifdef __IBMC__
+# if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC)
# pragma pack(1)
# else
# pragma pack(push, 1)
@@ -195,7 +205,11 @@ typedef struct { U64 v; } _PACKED U64_S;
typedef struct {size_t v;} _PACKED size_t_S;
#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# pragma pack(pop)
+# if defined(__SUNPRO_C) || defined(__SUNPRO_CC)
+# pragma pack(0)
+# else
+# pragma pack(pop)
+# endif
#endif
#define A16(x) (((U16_S *)(x))->v)
@@ -204,20 +218,26 @@ typedef struct {size_t v;} _PACKED size_t_S;
#define AARCH(x) (((size_t_S *)(x))->v)
-//**************************************
-// Constants
-//**************************************
-#define HASHTABLESIZE (1 << MEMORY_USAGE)
+/**************************************
+ Constants
+**************************************/
+#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
+#define HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
+#define HASH_SIZE_U32 (1 << LZ4_HASHLOG)
#define MINMATCH 4
#define COPYLENGTH 8
#define LASTLITERALS 5
#define MFLIMIT (COPYLENGTH+MINMATCH)
-#define MINLENGTH (MFLIMIT+1)
+static const int LZ4_minLength = (MFLIMIT+1);
+
+#define KB *(1U<<10)
+#define MB *(1U<<20)
+#define GB *(1U<<30)
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT-1))
-#define SKIPSTRENGTH 6 // Increasing this value will make the compression run slower on incompressible data
+#define LZ4_64KLIMIT ((64 KB) + (MFLIMIT-1))
+#define SKIPSTRENGTH 6 /* Increasing this value will make the compression run slower on incompressible data */
#define MAXD_LOG 16
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
@@ -228,44 +248,61 @@ typedef struct {size_t v;} _PACKED size_t_S;
#define RUN_MASK ((1U<<RUN_BITS)-1)
-//**************************************
-// Architecture-specific macros
-//**************************************
+/**************************************
+ Structures and local types
+**************************************/
+typedef struct {
+ U32 hashTable[HASH_SIZE_U32];
+ U32 currentOffset;
+ U32 initCheck;
+ const BYTE* dictionary;
+ const BYTE* bufferStart;
+ U32 dictSize;
+} LZ4_stream_t_internal;
+
+typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum { byPtr, byU32, byU16 } tableType_t;
+
+typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
+typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
+
+typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
+typedef enum { full = 0, partial = 1 } earlyEnd_directive;
+
+
+/**************************************
+ Architecture-specific macros
+**************************************/
#define STEPSIZE sizeof(size_t)
-#define LZ4_COPYSTEP(s,d) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }
-#define LZ4_COPY8(s,d) { LZ4_COPYSTEP(s,d); if (STEPSIZE<8) LZ4_COPYSTEP(s,d); }
-#define LZ4_SECURECOPY(s,d,e) { if ((STEPSIZE==8)&&(d<e)) LZ4_WILDCOPY(s,d,e); }
-
-#if LZ4_ARCH64 // 64-bit
-# define HTYPE U32
-# define INITBASE(base) const BYTE* const base = ip
-#else // 32-bit
-# define HTYPE const BYTE*
-# define INITBASE(base) const int base = 0
-#endif
+#define LZ4_COPYSTEP(d,s) { AARCH(d) = AARCH(s); d+=STEPSIZE; s+=STEPSIZE; }
+#define LZ4_COPY8(d,s) { LZ4_COPYSTEP(d,s); if (STEPSIZE<8) LZ4_COPYSTEP(d,s); }
#if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE))
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; }
# define LZ4_WRITE_LITTLEENDIAN_16(p,i) { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p+=2; }
-#else // Little Endian
+#else /* Little Endian */
# define LZ4_READ_LITTLEENDIAN_16(d,s,p) { d = (s) - A16(p); }
# define LZ4_WRITE_LITTLEENDIAN_16(p,v) { A16(p) = v; p+=2; }
#endif
-//**************************************
-// Macros
-//**************************************
-#define LZ4_WILDCOPY(s,d,e) { do { LZ4_COPY8(s,d) } while (d<e); }
-#define LZ4_BLINDCOPY(s,d,l) { BYTE* e=(d)+(l); LZ4_WILDCOPY(s,d,e); d=e; }
+/**************************************
+ Macros
+**************************************/
+#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
+#if LZ4_ARCH64 || !defined(__GNUC__)
+# define LZ4_WILDCOPY(d,s,e) { do { LZ4_COPY8(d,s) } while (d<e); } /* at the end, d>=e; */
+#else
+# define LZ4_WILDCOPY(d,s,e) { if (likely(e-d <= 8)) LZ4_COPY8(d,s) else do { LZ4_COPY8(d,s) } while (d<e); }
+#endif
-//****************************
-// Private functions
-//****************************
+/****************************
+ Private local functions
+****************************/
#if LZ4_ARCH64
-forceinline int LZ4_NbCommonBytes (register U64 val)
+static int LZ4_NbCommonBytes (register U64 val)
{
# if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -297,7 +334,7 @@ forceinline int LZ4_NbCommonBytes (register U64 val)
#else
-forceinline int LZ4_NbCommonBytes (register U32 val)
+static int LZ4_NbCommonBytes (register U32 val)
{
# if defined(LZ4_BIG_ENDIAN)
# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
@@ -329,227 +366,530 @@ forceinline int LZ4_NbCommonBytes (register U32 val)
#endif
+/********************************
+ Compression functions
+********************************/
+int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
+int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-//******************************
-// Compression functions
-//******************************
+static int LZ4_hashSequence(U32 sequence, tableType_t tableType)
+{
+ if (tableType == byU16)
+ return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
+ else
+ return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
+}
-/*
-int LZ4_compress_stack(
- const char* source,
- char* dest,
- int inputSize)
+static int LZ4_hashPosition(const BYTE* p, tableType_t tableType) { return LZ4_hashSequence(A32(p), tableType); }
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest'.
-Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize).
-return : the number of bytes written in buffer 'dest'
-*/
-#define FUNCTION_NAME LZ4_compress_stack
-#include "lz4_encoder.h"
+static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ switch (tableType)
+ {
+ case byPtr: { const BYTE** hashTable = (const BYTE**) tableBase; hashTable[h] = p; break; }
+ case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); break; }
+ case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); break; }
+ }
+}
+static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ U32 h = LZ4_hashPosition(p, tableType);
+ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
+}
-/*
-int LZ4_compress_stack_limitedOutput(
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize)
+static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
+ if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
+ { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
+}
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
-If it cannot achieve it, compression will stop, and result of the function will be zero.
-return : the number of bytes written in buffer 'dest', or 0 if the compression fails
-*/
-#define FUNCTION_NAME LZ4_compress_stack_limitedOutput
-#define LIMITED_OUTPUT
-#include "lz4_encoder.h"
+static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
+{
+ U32 h = LZ4_hashPosition(p, tableType);
+ return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
+}
+static unsigned LZ4_count(const BYTE* pIn, const BYTE* pRef, const BYTE* pInLimit)
+{
+ const BYTE* const pStart = pIn;
-/*
-int LZ4_compress64k_stack(
- const char* source,
- char* dest,
- int inputSize)
+ while (likely(pIn<pInLimit-(STEPSIZE-1)))
+ {
+ size_t diff = AARCH(pRef) ^ AARCH(pIn);
+ if (!diff) { pIn+=STEPSIZE; pRef+=STEPSIZE; continue; }
+ pIn += LZ4_NbCommonBytes(diff);
+ return (unsigned)(pIn - pStart);
+ }
+ if (LZ4_64BITS) if ((pIn<(pInLimit-3)) && (A32(pRef) == A32(pIn))) { pIn+=4; pRef+=4; }
+ if ((pIn<(pInLimit-1)) && (A16(pRef) == A16(pIn))) { pIn+=2; pRef+=2; }
+ if ((pIn<pInLimit) && (*pRef == *pIn)) pIn++;
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest'.
-This function compresses better than LZ4_compress_stack(), on the condition that
-'inputSize' must be < to LZ4_64KLIMIT, or the function will fail.
-Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize).
-return : the number of bytes written in buffer 'dest', or 0 if compression fails
-*/
-#define FUNCTION_NAME LZ4_compress64k_stack
-#define COMPRESS_64K
-#include "lz4_encoder.h"
+ return (unsigned)(pIn - pStart);
+}
-/*
-int LZ4_compress64k_stack_limitedOutput(
+static int LZ4_compress_generic(
+ void* ctx,
const char* source,
char* dest,
int inputSize,
- int maxOutputSize)
+ int maxOutputSize,
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
-This function compresses better than LZ4_compress_stack_limitedOutput(), on the condition that
-'inputSize' must be < to LZ4_64KLIMIT, or the function will fail.
-If it cannot achieve it, compression will stop, and result of the function will be zero.
-return : the number of bytes written in buffer 'dest', or 0 if the compression fails
-*/
-#define FUNCTION_NAME LZ4_compress64k_stack_limitedOutput
-#define COMPRESS_64K
-#define LIMITED_OUTPUT
-#include "lz4_encoder.h"
+ limitedOutput_directive outputLimited,
+ tableType_t tableType,
+ dict_directive dict,
+ dictIssue_directive dictIssue)
+{
+ LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx;
+
+ const BYTE* ip = (const BYTE*) source;
+ const BYTE* base;
+ const BYTE* lowLimit;
+ const BYTE* const lowRefLimit = ip - dictPtr->dictSize;
+ const BYTE* const dictionary = dictPtr->dictionary;
+ const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
+ const size_t dictDelta = dictEnd - (const BYTE*)source;
+ const BYTE* anchor = (const BYTE*) source;
+ const BYTE* const iend = ip + inputSize;
+ const BYTE* const mflimit = iend - MFLIMIT;
+ const BYTE* const matchlimit = iend - LASTLITERALS;
+ BYTE* op = (BYTE*) dest;
+ BYTE* const olimit = op + maxOutputSize;
-/*
-void* LZ4_createHeapMemory();
-int LZ4_freeHeapMemory(void* ctx);
+ const int skipStrength = SKIPSTRENGTH;
+ U32 forwardH;
+ size_t refDelta=0;
-Used to allocate and free hashTable memory
-to be used by the LZ4_compress_heap* family of functions.
-LZ4_createHeapMemory() returns NULL is memory allocation fails.
-*/
-void* LZ4_create() { return malloc(HASHTABLESIZE); }
-int LZ4_free(void* ctx) { free(ctx); return 0; }
+ /* Init conditions */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
+ switch(dict)
+ {
+ case noDict:
+ default:
+ base = (const BYTE*)source;
+ lowLimit = (const BYTE*)source;
+ break;
+ case withPrefix64k:
+ base = (const BYTE*)source - dictPtr->currentOffset;
+ lowLimit = (const BYTE*)source - dictPtr->dictSize;
+ break;
+ case usingExtDict:
+ base = (const BYTE*)source - dictPtr->currentOffset;
+ lowLimit = (const BYTE*)source;
+ break;
+ }
+ if ((tableType == byU16) && (inputSize>=(int)LZ4_64KLIMIT)) return 0; /* Size too large (not within 64K limit) */
+ if (inputSize<LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
+ /* First Byte */
+ LZ4_putPosition(ip, ctx, tableType, base);
+ ip++; forwardH = LZ4_hashPosition(ip, tableType);
-/*
-int LZ4_compress_heap(
- void* ctx,
- const char* source,
- char* dest,
- int inputSize)
+ /* Main Loop */
+ for ( ; ; )
+ {
+ const BYTE* ref;
+ BYTE* token;
+ {
+ const BYTE* forwardIp = ip;
+ unsigned step=1;
+ unsigned searchMatchNb = (1U << skipStrength);
+
+ /* Find a match */
+ do {
+ U32 h = forwardH;
+ ip = forwardIp;
+ forwardIp += step;
+ step = searchMatchNb++ >> skipStrength;
+
+ if (unlikely(forwardIp > mflimit)) goto _last_literals;
+
+ ref = LZ4_getPositionOnHash(h, ctx, tableType, base);
+ if (dict==usingExtDict)
+ {
+ if (ref<(const BYTE*)source)
+ {
+ refDelta = dictDelta;
+ lowLimit = dictionary;
+ }
+ else
+ {
+ refDelta = 0;
+ lowLimit = (const BYTE*)source;
+ }
+ }
+ forwardH = LZ4_hashPosition(forwardIp, tableType);
+ LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
+
+ } while ( ((dictIssue==dictSmall) ? (ref < lowRefLimit) : 0)
+ || ((tableType==byU16) ? 0 : (ref + MAX_DISTANCE < ip))
+ || (A32(ref+refDelta) != A32(ip)) );
+ }
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest'.
-The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'.
-Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize).
-return : the number of bytes written in buffer 'dest'
-*/
-#define FUNCTION_NAME LZ4_compress_heap
-#define USE_HEAPMEMORY
-#include "lz4_encoder.h"
+ /* Catch up */
+ while ((ip>anchor) && (ref+refDelta > lowLimit) && (unlikely(ip[-1]==ref[refDelta-1]))) { ip--; ref--; }
+ {
+ /* Encode Literal length */
+ unsigned litLength = (unsigned)(ip - anchor);
+ token = op++;
+ if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
+ return 0; /* Check output limit */
+ if (litLength>=RUN_MASK)
+ {
+ int len = (int)litLength-RUN_MASK;
+ *token=(RUN_MASK<<ML_BITS);
+ for(; len >= 255 ; len-=255) *op++ = 255;
+ *op++ = (BYTE)len;
+ }
+ else *token = (BYTE)(litLength<<ML_BITS);
-/*
-int LZ4_compress_heap_limitedOutput(
- void* ctx,
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize)
+ /* Copy Literals */
+ { BYTE* end = op+litLength; LZ4_WILDCOPY(op,anchor,end); op=end; }
+ }
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
-If it cannot achieve it, compression will stop, and result of the function will be zero.
-The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'.
-return : the number of bytes written in buffer 'dest', or 0 if the compression fails
-*/
-#define FUNCTION_NAME LZ4_compress_heap_limitedOutput
-#define LIMITED_OUTPUT
-#define USE_HEAPMEMORY
-#include "lz4_encoder.h"
+_next_match:
+ /* Encode Offset */
+ LZ4_WRITE_LITTLEENDIAN_16(op, (U16)(ip-ref));
+ /* Encode MatchLength */
+ {
+ unsigned matchLength;
-/*
-int LZ4_compress64k_heap(
- void* ctx,
- const char* source,
- char* dest,
- int inputSize)
+ if ((dict==usingExtDict) && (lowLimit==dictionary))
+ {
+ const BYTE* limit;
+ ref += refDelta;
+ limit = ip + (dictEnd-ref);
+ if (limit > matchlimit) limit = matchlimit;
+ matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, limit);
+ ip += MINMATCH + matchLength;
+ if (ip==limit)
+ {
+ unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit);
+ matchLength += more;
+ ip += more;
+ }
+ }
+ else
+ {
+ matchLength = LZ4_count(ip+MINMATCH, ref+MINMATCH, matchlimit);
+ ip += MINMATCH + matchLength;
+ }
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest'.
-The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'.
-'inputSize' must be < to LZ4_64KLIMIT, or the function will fail.
-Destination buffer must be already allocated, and sized at a minimum of LZ4_compressBound(inputSize).
-return : the number of bytes written in buffer 'dest'
-*/
-#define FUNCTION_NAME LZ4_compress64k_heap
-#define COMPRESS_64K
-#define USE_HEAPMEMORY
-#include "lz4_encoder.h"
+ if (matchLength>=ML_MASK)
+ {
+ if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit)))
+ return 0; /* Check output limit */
+ *token += ML_MASK;
+ matchLength -= ML_MASK;
+ for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; }
+ if (matchLength >= 255) { matchLength-=255; *op++ = 255; }
+ *op++ = (BYTE)matchLength;
+ }
+ else *token += (BYTE)(matchLength);
+ }
+ anchor = ip;
-/*
-int LZ4_compress64k_heap_limitedOutput(
- void* ctx,
- const char* source,
- char* dest,
- int inputSize,
- int maxOutputSize)
+ /* Test end of chunk */
+ if (ip > mflimit) break;
-Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
-If it cannot achieve it, compression will stop, and result of the function will be zero.
-The memory used for compression must be created by LZ4_createHeapMemory() and provided by pointer 'ctx'.
-'inputSize' must be < to LZ4_64KLIMIT, or the function will fail.
-return : the number of bytes written in buffer 'dest', or 0 if the compression fails
-*/
-#define FUNCTION_NAME LZ4_compress64k_heap_limitedOutput
-#define COMPRESS_64K
-#define LIMITED_OUTPUT
-#define USE_HEAPMEMORY
-#include "lz4_encoder.h"
+ /* Fill table */
+ LZ4_putPosition(ip-2, ctx, tableType, base);
+
+ /* Test next position */
+ ref = LZ4_getPosition(ip, ctx, tableType, base);
+ if (dict==usingExtDict)
+ {
+ if (ref<(const BYTE*)source)
+ {
+ refDelta = dictDelta;
+ lowLimit = dictionary;
+ }
+ else
+ {
+ refDelta = 0;
+ lowLimit = (const BYTE*)source;
+ }
+ }
+ LZ4_putPosition(ip, ctx, tableType, base);
+ if ( ((dictIssue==dictSmall) ? (ref>=lowRefLimit) : 1)
+ && (ref+MAX_DISTANCE>=ip)
+ && (A32(ref+refDelta)==A32(ip)) )
+ { token=op++; *token=0; goto _next_match; }
+
+ /* Prepare next loop */
+ forwardH = LZ4_hashPosition(++ip, tableType);
+ }
+
+_last_literals:
+ /* Encode Last Literals */
+ {
+ int lastRun = (int)(iend - anchor);
+ if ((outputLimited) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
+ return 0; /* Check output limit */
+ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun >= 255 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+ else *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend-anchor;
+ }
+
+ /* End */
+ return (int) (((char*)op)-dest);
+}
int LZ4_compress(const char* source, char* dest, int inputSize)
{
-#if HEAPMODE
- void* ctx = LZ4_create();
+#if (HEAPMODE)
+ void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */
+#else
+ U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
+#endif
int result;
- if (ctx == NULL) return 0; // Failed allocation => compression not done
- if (inputSize < LZ4_64KLIMIT)
- result = LZ4_compress64k_heap(ctx, source, dest, inputSize);
- else result = LZ4_compress_heap(ctx, source, dest, inputSize);
- LZ4_free(ctx);
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue);
+ else
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue);
+
+#if (HEAPMODE)
+ FREEMEM(ctx);
+#endif
return result;
+}
+
+int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+#if (HEAPMODE)
+ void* ctx = ALLOCATOR(LZ4_STREAMSIZE_U32, 4); /* Aligned on 4-bytes boundaries */
#else
- if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress64k_stack(source, dest, inputSize);
- return LZ4_compress_stack(source, dest, inputSize);
+ U32 ctx[LZ4_STREAMSIZE_U32] = {0}; /* Ensure data is aligned on 4-bytes boundaries */
+#endif
+ int result;
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue);
+ else
+ result = LZ4_compress_generic((void*)ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue);
+
+#if (HEAPMODE)
+ FREEMEM(ctx);
#endif
+ return result;
}
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize)
+/*****************************************
+ Experimental : Streaming functions
+*****************************************/
+
+/*
+ * LZ4_initStream
+ * Use this function once, to init a newly allocated LZ4_stream_t structure
+ * Return : 1 if OK, 0 if error
+ */
+void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
+{
+ MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
+}
+
+LZ4_stream_t* LZ4_createStream(void)
+{
+ LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(4, LZ4_STREAMSIZE_U32);
+ LZ4_resetStream(lz4s);
+ return lz4s;
+}
+
+int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
+{
+ FREEMEM(LZ4_stream);
+ return (0);
+}
+
+
+int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
+{
+ LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
+ const BYTE* p = (const BYTE*)dictionary;
+ const BYTE* const dictEnd = p + dictSize;
+ const BYTE* base;
+
+ LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
+ if (dict->initCheck) LZ4_resetStream(LZ4_dict); /* Uninitialized structure detected */
+
+ if (dictSize < MINMATCH)
+ {
+ dict->dictionary = NULL;
+ dict->dictSize = 0;
+ return 1;
+ }
+
+ if (p <= dictEnd - 64 KB) p = dictEnd - 64 KB;
+ base = p - dict->currentOffset;
+ dict->dictionary = p;
+ dict->dictSize = (U32)(dictEnd - p);
+ dict->currentOffset += dict->dictSize;
+
+ while (p <= dictEnd-MINMATCH)
+ {
+ LZ4_putPosition(p, dict, byU32, base);
+ p+=3;
+ }
+
+ return 1;
+}
+
+
+static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
{
-#if HEAPMODE
- void* ctx = LZ4_create();
+ if ((LZ4_dict->currentOffset > 0x80000000) ||
+ ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */
+ {
+ /* rescale hash table */
+ U32 delta = LZ4_dict->currentOffset - 64 KB;
+ const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
+ int i;
+ for (i=0; i<HASH_SIZE_U32; i++)
+ {
+ if (LZ4_dict->hashTable[i] < delta) LZ4_dict->hashTable[i]=0;
+ else LZ4_dict->hashTable[i] -= delta;
+ }
+ LZ4_dict->currentOffset = 64 KB;
+ if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB;
+ LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize;
+ }
+}
+
+
+FORCE_INLINE int LZ4_compress_continue_generic (void* LZ4_stream, const char* source, char* dest, int inputSize,
+ int maxOutputSize, limitedOutput_directive limit)
+{
+ LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
+ const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+
+ const BYTE* smallest = (const BYTE*) source;
+ if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */
+ if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
+ LZ4_renormDictT(streamPtr, smallest);
+
+ /* Check overlapping input/dictionary space */
+ {
+ const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+ if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
+ {
+ streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
+ if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
+ if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
+ streamPtr->dictionary = dictEnd - streamPtr->dictSize;
+ }
+ }
+
+ /* prefix mode : source data follows dictionary */
+ if (dictEnd == (const BYTE*)source)
+ {
+ int result;
+ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, dictSmall);
+ else
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, withPrefix64k, noDictIssue);
+ streamPtr->dictSize += (U32)inputSize;
+ streamPtr->currentOffset += (U32)inputSize;
+ return result;
+ }
+
+ /* external dictionary mode */
+ {
+ int result;
+ if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, dictSmall);
+ else
+ result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limit, byU32, usingExtDict, noDictIssue);
+ streamPtr->dictionary = (const BYTE*)source;
+ streamPtr->dictSize = (U32)inputSize;
+ streamPtr->currentOffset += (U32)inputSize;
+ return result;
+ }
+}
+
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize)
+{
+ return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, 0, notLimited);
+}
+
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ return LZ4_compress_continue_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput);
+}
+
+
+/* Hidden debug function, to force separate dictionary mode */
+int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
+{
+ LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict;
int result;
- if (ctx == NULL) return 0; // Failed allocation => compression not done
- if (inputSize < LZ4_64KLIMIT)
- result = LZ4_compress64k_heap_limitedOutput(ctx, source, dest, inputSize, maxOutputSize);
- else result = LZ4_compress_heap_limitedOutput(ctx, source, dest, inputSize, maxOutputSize);
- LZ4_free(ctx);
+ const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
+
+ const BYTE* smallest = dictEnd;
+ if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
+ LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest);
+
+ result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue);
+
+ streamPtr->dictionary = (const BYTE*)source;
+ streamPtr->dictSize = (U32)inputSize;
+ streamPtr->currentOffset += (U32)inputSize;
+
return result;
-#else
- if (inputSize < (int)LZ4_64KLIMIT) return LZ4_compress64k_stack_limitedOutput(source, dest, inputSize, maxOutputSize);
- return LZ4_compress_stack_limitedOutput(source, dest, inputSize, maxOutputSize);
-#endif
}
-//****************************
-// Decompression functions
-//****************************
+int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
+{
+ LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
+ const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
-typedef enum { noPrefix = 0, withPrefix = 1 } prefix64k_directive;
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
-typedef enum { full = 0, partial = 1 } earlyEnd_directive;
+ if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
+ if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
+ memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
-// This generic decompression function cover all use cases.
-// It shall be instanciated several times, using different sets of directives
-// Note that it is essential this generic function is really inlined,
-// in order to remove useless branches during compilation optimisation.
-forceinline int LZ4_decompress_generic(
+ dict->dictionary = (const BYTE*)safeBuffer;
+ dict->dictSize = (U32)dictSize;
+
+ return dictSize;
+}
+
+
+
+/****************************
+ Decompression functions
+****************************/
+/*
+ * This generic decompression function cover all use cases.
+ * It shall be instantiated several times, using different sets of directives
+ * Note that it is essential this generic function is really inlined,
+ * in order to remove useless branches during compilation optimization.
+ */
+FORCE_INLINE int LZ4_decompress_generic(
const char* source,
char* dest,
- int inputSize, //
- int outputSize, // OutputSize must be != 0; if endOnInput==endOnInputSize, this value is the max size of Output Buffer.
-
- int endOnInput, // endOnOutputSize, endOnInputSize
- int prefix64k, // noPrefix, withPrefix
- int partialDecoding, // full, partial
- int targetOutputSize // only used if partialDecoding==partial
+ int inputSize,
+ int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
+
+ int endOnInput, /* endOnOutputSize, endOnInputSize */
+ int partialDecoding, /* full, partial */
+ int targetOutputSize, /* only used if partialDecoding==partial */
+ int dict, /* noDict, withPrefix64k, usingExtDict */
+ const char* dictStart, /* only if dict==usingExtDict */
+ int dictSize /* note : = 0 if noDict */
)
{
- // Local Variables
+ /* Local Variables */
const BYTE* restrict ip = (const BYTE*) source;
const BYTE* ref;
const BYTE* const iend = ip + inputSize;
@@ -558,139 +898,370 @@ forceinline int LZ4_decompress_generic(
BYTE* const oend = op + outputSize;
BYTE* cpy;
BYTE* oexit = op + targetOutputSize;
+ const BYTE* const lowLimit = (const BYTE*)dest - dictSize;
- size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
- size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
-#endif
+ const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
+ const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4};
+ const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
+ const int safeDecode = (endOnInput==endOnInputSize);
+ const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
- // Special cases
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; // targetOutputSize too high => decode everything
- if ((endOnInput) && unlikely(outputSize==0)) return ((inputSize==1) && (*ip==0)) ? 0 : -1; // Empty output buffer
- if ((!endOnInput) && unlikely(outputSize==0)) return (*ip==0?1:-1);
+ /* Special cases */
+ if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
+ if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
+ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
- // Main Loop
+
+ /* Main Loop */
while (1)
{
unsigned token;
size_t length;
- // get runlength
+ /* get runlength */
token = *ip++;
if ((length=(token>>ML_BITS)) == RUN_MASK)
- {
- unsigned s=255;
- while (((endOnInput)?ip<iend:1) && (s==255))
- {
- s = *ip++;
- length += s;
- }
+ {
+ unsigned s;
+ do
+ {
+ s = *ip++;
+ length += s;
+ }
+ while (likely((endOnInput)?ip<iend-RUN_MASK:1) && (s==255));
+ if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)(op))) goto _output_error; /* overflow detection */
+ if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(ip+length)<(size_t)(ip))) goto _output_error; /* overflow detection */
}
- // copy literals
+ /* copy literals */
cpy = op+length;
if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
|| ((!endOnInput) && (cpy>oend-COPYLENGTH)))
{
if (partialDecoding)
{
- if (cpy > oend) goto _output_error; // Error : write attempt beyond end of output buffer
- if ((endOnInput) && (ip+length > iend)) goto _output_error; // Error : read attempt beyond end of input buffer
+ if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */
+ if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
}
else
{
- if ((!endOnInput) && (cpy != oend)) goto _output_error; // Error : block decoding must stop exactly there
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; // Error : input must be consumed
+ if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
}
memcpy(op, ip, length);
ip += length;
op += length;
- break; // Necessarily EOF, due to parsing restrictions
+ break; /* Necessarily EOF, due to parsing restrictions */
}
- LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy;
+ LZ4_WILDCOPY(op, ip, cpy); ip -= (op-cpy); op = cpy;
- // get offset
+ /* get offset */
LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2;
- if ((prefix64k==noPrefix) && unlikely(ref < (BYTE* const)dest)) goto _output_error; // Error : offset outside destination buffer
+ if ((checkOffset) && (unlikely(ref < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
+
+ /* get matchlength */
+ if ((length=(token&ML_MASK)) == ML_MASK)
+ {
+ unsigned s;
+ do
+ {
+ if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
+ s = *ip++;
+ length += s;
+ } while (s==255);
+ if ((safeDecode) && LZ4_32BITS && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
+ }
+ length += MINMATCH;
+
+ /* check external dictionary */
+ if ((dict==usingExtDict) && (ref < (BYTE* const)dest))
+ {
+ if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error;
- // get matchlength
- if ((length=(token&ML_MASK)) == ML_MASK)
- {
- for ( ; (!endOnInput) || (ip<iend-(LASTLITERALS+1)) ; ) // Ensure enough bytes remain for LASTLITERALS + token
+ if (length <= (size_t)(dest-(char*)ref))
{
- unsigned s = *ip++;
- length += s;
- if (s==255) continue;
- break;
+ ref = dictEnd - (dest-(char*)ref);
+ memcpy(op, ref, length);
+ op += length;
}
+ else
+ {
+ size_t copySize = (size_t)(dest-(char*)ref);
+ memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+ copySize = length - copySize;
+ if (copySize > (size_t)((char*)op-dest)) /* overlap */
+ {
+ BYTE* const endOfMatch = op + copySize;
+ const BYTE* copyFrom = (BYTE*)dest;
+ while (op < endOfMatch) *op++ = *copyFrom++;
+ }
+ else
+ {
+ memcpy(op, dest, copySize);
+ op += copySize;
+ }
+ }
+ continue;
}
- // copy repeated sequence
- if unlikely((op-ref)<(int)STEPSIZE)
+ /* copy repeated sequence */
+ cpy = op + length;
+ if (unlikely((op-ref)<(int)STEPSIZE))
{
-#if LZ4_ARCH64
- size_t dec64 = dec64table[op-ref];
-#else
- const size_t dec64 = 0;
-#endif
+ const size_t dec64 = dec64table[op-ref];
op[0] = ref[0];
op[1] = ref[1];
op[2] = ref[2];
op[3] = ref[3];
- op += 4, ref += 4; ref -= dec32table[op-ref];
- A32(op) = A32(ref);
- op += STEPSIZE-4; ref -= dec64;
- } else { LZ4_COPYSTEP(ref,op); }
- cpy = op + length - (STEPSIZE-4);
+ ref += dec32table[op-ref];
+ A32(op+4) = A32(ref);
+ op += 8; ref -= dec64;
+ } else { LZ4_COPY8(op,ref); }
- if unlikely(cpy>oend-(COPYLENGTH)-(STEPSIZE-4))
+ if (unlikely(cpy>oend-12))
{
- if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals
- LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH));
+ if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last 5 bytes must be literals */
+ if (op<oend-COPYLENGTH) LZ4_WILDCOPY(op, ref, (oend-COPYLENGTH));
while(op<cpy) *op++=*ref++;
- op=cpy;
- continue;
}
- LZ4_WILDCOPY(ref, op, cpy);
- op=cpy; // correction
+ else LZ4_WILDCOPY(op, ref, cpy);
+ op=cpy; /* correction */
}
- // end of decoding
+ /* end of decoding */
if (endOnInput)
- return (int) (((char*)op)-dest); // Nb of output bytes decoded
+ return (int) (((char*)op)-dest); /* Nb of output bytes decoded */
else
- return (int) (((char*)ip)-source); // Nb of input bytes read
+ return (int) (((char*)ip)-source); /* Nb of input bytes read */
- // Overflow error detected
+ /* Overflow error detected */
_output_error:
return (int) (-(((char*)ip)-source))-1;
}
-int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize)
+int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, NULL, 0);
+}
+
+int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, NULL, 0);
+}
+
+int LZ4_decompress_fast(const char* source, char* dest, int originalSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
+}
+
+/* streaming decompression functions */
+
+typedef struct
+{
+ const char* dictionary;
+ int dictSize;
+} LZ4_streamDecode_t_internal;
+
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode()
+ * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
+ */
+LZ4_streamDecode_t* LZ4_createStreamDecode(void)
+{
+ LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(sizeof(U32), LZ4_STREAMDECODESIZE_U32);
+ MEM_INIT(lz4s, 0, LZ4_STREAMDECODESIZE);
+ return lz4s;
+}
+
+int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream)
+{
+ FREEMEM(LZ4_stream);
+ return 0;
+}
+
+/*
+ * LZ4_setStreamDecode
+ * Use this function to instruct where to find the dictionary
+ * This function is not necessary if previous data is still available where it was decoded.
+ * Loading a size of 0 is allowed (same effect as no dictionary).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize)
+{
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ lz4sd->dictionary = dictionary;
+ lz4sd->dictSize = dictSize;
+ return 1;
+}
+
+/*
+*_continue() :
+ These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks must still be available at the memory position where they were decoded.
+ If it's not possible, save the relevant part of decoded data into a safe buffer,
+ and indicate where it stands using LZ4_setDictDecode()
+*/
+int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
+{
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ int result;
+
+ result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
+ if (result <= 0) return result;
+ if (lz4sd->dictionary + lz4sd->dictSize == dest)
+ {
+ lz4sd->dictSize += result;
+ }
+ else
+ {
+ lz4sd->dictionary = dest;
+ lz4sd->dictSize = result;
+ }
+
+ return result;
+}
+
+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
+{
+ LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
+ int result;
+
+ result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->dictionary, lz4sd->dictSize);
+ if (result <= 0) return result;
+ if (lz4sd->dictionary + lz4sd->dictSize == dest)
+ {
+ lz4sd->dictSize += result;
+ }
+ else
+ {
+ lz4sd->dictionary = dest;
+ lz4sd->dictSize = result;
+ }
+
+ return result;
+}
+
+
+/*
+Advanced decoding functions :
+*_usingDict() :
+ These decoding functions work the same as "_continue" ones,
+ the dictionary must be explicitly provided within parameters
+*/
+
+FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
+{
+ if (dictSize==0)
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, NULL, 64 KB);
+ if ((dictStart+dictSize == dest) && (dictSize >= (int)(64 KB - 1)))
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, NULL, 64 KB);
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, dictStart, dictSize);
+}
+
+int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, full, 0);
+ //return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
+ return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
}
-int LZ4_decompress_fast(const char* source, char* dest, int outputSize)
+int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
{
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, noPrefix, full, 0);
+ //return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, dictStart, dictSize);
+ return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
}
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int inputSize, int maxOutputSize)
+/* debug function */
+int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, dictStart, dictSize);
+}
+
+
+/***************************************************
+ Obsolete Functions
+***************************************************/
+/*
+These function names are deprecated and should no longer be used.
+They are only provided here for compatibility with older user programs.
+- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
+- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
+*/
+int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
+int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
+
+
+/* Obsolete Streaming functions */
+
+int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
+
+static void LZ4_init(LZ4_stream_t_internal* lz4ds, const BYTE* base)
{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, withPrefix, full, 0);
+ MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE);
+ lz4ds->bufferStart = base;
}
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int outputSize)
+int LZ4_resetStreamState(void* state, const char* inputBuffer)
{
- return LZ4_decompress_generic(source, dest, 0, outputSize, endOnOutputSize, withPrefix, full, 0);
+ if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
+ LZ4_init((LZ4_stream_t_internal*)state, (const BYTE*)inputBuffer);
+ return 0;
}
-int LZ4_decompress_safe_partial(const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize)
+void* LZ4_create (const char* inputBuffer)
{
- return LZ4_decompress_generic(source, dest, inputSize, maxOutputSize, endOnInputSize, noPrefix, partial, targetOutputSize);
+ void* lz4ds = ALLOCATOR(4, LZ4_STREAMSIZE_U32);
+ LZ4_init ((LZ4_stream_t_internal*)lz4ds, (const BYTE*)inputBuffer);
+ return lz4ds;
}
+char* LZ4_slideInputBuffer (void* LZ4_Data)
+{
+ LZ4_stream_t_internal* lz4ds = (LZ4_stream_t_internal*)LZ4_Data;
+
+ LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)lz4ds->bufferStart, 64 KB);
+
+ return (char*)(lz4ds->bufferStart + 64 KB);
+}
+
+/* Obsolete compresson functions using User-allocated state */
+
+int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
+
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize)
+{
+ if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
+ MEM_INIT(state, 0, LZ4_STREAMSIZE);
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue);
+ else
+ return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue);
+}
+
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+ if (((size_t)(state)&3) != 0) return 0; /* Error : state is not aligned on 4-bytes boundary */
+ MEM_INIT(state, 0, LZ4_STREAMSIZE);
+
+ if (inputSize < (int)LZ4_64KLIMIT)
+ return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue);
+ else
+ return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64BITS ? byU32 : byPtr, noDict, noDictIssue);
+}
+
+/* Obsolete streaming decompression functions */
+
+int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, NULL, 64 KB);
+}
+
+int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, NULL, 64 KB);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97fa1fc3/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h
index d866234..44ada14 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.h
@@ -1,7 +1,7 @@
/*
LZ4 - Fast LZ compression algorithm
Header File
- Copyright (C) 2011-2013, Yann Collet.
+ Copyright (C) 2011-2014, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -28,151 +28,295 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- LZ4 source repository : http://code.google.com/p/lz4/
+ - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
-
-/*
- Copied from:
- URL: http://lz4.googlecode.com/svn/trunk
- Repository Root: http://lz4.googlecode.com/svn
- Repository UUID: 650e7d94-2a16-8b24-b05c-7c0b3f6821cd
- Revision: 99
- Node Kind: directory
- Schedule: normal
- Last Changed Author: yann.collet.73@gmail.com
- Last Changed Rev: 99
- Last Changed Date: 2013-07-27 19:19:31 +0800 (Sat, 27 Jul 2013)
- */
-
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
+/*
+ * lz4.h provides raw compression format functions, for optimal performance and integration into programs.
+ * If you need to generate data using an inter-operable format (respecting the framing specification),
+ * please use lz4frame.h instead.
+*/
-//**************************************
-// Compiler Options
-//**************************************
-#if defined(_MSC_VER) && !defined(__cplusplus) // Visual Studio
-# define inline __inline // Visual C is not C99, but supports some kind of inline
-#endif
+/**************************************
+ Version
+**************************************/
+#define LZ4_VERSION_MAJOR 1 /* for major interface/format changes */
+#define LZ4_VERSION_MINOR 3 /* for minor interface/format changes */
+#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
+int LZ4_versionNumber (void);
+
+/**************************************
+ Tuning parameter
+**************************************/
+/*
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
+#define LZ4_MEMORY_USAGE 14
-//****************************
-// Simple Functions
-//****************************
+/**************************************
+ Simple Functions
+**************************************/
-int LZ4_compress (const char* source, char* dest, int inputSize);
-int LZ4_decompress_safe (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compress (const char* source, char* dest, int sourceSize);
+int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
LZ4_compress() :
- Compresses 'inputSize' bytes from 'source' into 'dest'.
+ Compresses 'sourceSize' bytes from 'source' into 'dest'.
Destination buffer must be already allocated,
and must be sized to handle worst cases situations (input data not compressible)
Worst case size evaluation is provided by function LZ4_compressBound()
- inputSize : Max supported value is ~1.9GB
+ inputSize : Max supported value is LZ4_MAX_INPUT_SIZE
return : the number of bytes written in buffer dest
or 0 if the compression fails
LZ4_decompress_safe() :
- maxOutputSize : is the size of the destination buffer (which must be already allocated)
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ compressedSize : is obviously the source size
+ maxDecompressedSize : is the size of the destination buffer, which must be already allocated.
+ return : the number of bytes decompressed into the destination buffer (necessarily <= maxDecompressedSize)
+ If the destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against buffer overflow exploits (never writes outside of output buffer, and never reads outside of input buffer). Therefore, it is protected against malicious data packets
+ This function is protected against buffer overflow exploits,
+ and never writes outside of output buffer, nor reads outside of input buffer.
+ It is also protected against malicious data packets.
*/
-//****************************
-// Advanced Functions
-//****************************
-
-static inline int LZ4_compressBound(int isize) { return ((isize) + ((isize)/255) + 16); }
-#define LZ4_COMPRESSBOUND( isize) ((isize) + ((isize)/255) + 16)
+/**************************************
+ Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize) ((unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
/*
LZ4_compressBound() :
- Provides the maximum size that LZ4 may output in a "worst case" scenario (input data not compressible)
- primarily useful for memory allocation of output buffer.
- inline function is recommended for the general case,
- macro is also provided when result needs to be evaluated at compilation (such as table size allocation).
+ Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
+ This function is primarily useful for memory allocation purposes (output buffer size).
+ Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
- isize : is the input size. Max supported value is ~1.9GB
+ isize : is the input size. Max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
- note : this function is limited by "int" range (2^31-1)
+ or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
*/
+int LZ4_compressBound(int isize);
-int LZ4_compress_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
-
/*
LZ4_compress_limitedOutput() :
- Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
+ Compress 'sourceSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
If it cannot achieve it, compression will stop, and result of the function will be zero.
+ This saves time and memory on detecting non-compressible (or barely compressible) data.
This function never writes outside of provided output buffer.
- inputSize : Max supported value is ~1.9GB
+ sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxOutputSize : is the size of the destination buffer (which must be already allocated)
return : the number of bytes written in buffer 'dest'
- or 0 if the compression fails
+ or 0 if compression fails
*/
+int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+
+/*
+LZ4_compress_withState() :
+ Same compression functions, but using an externally allocated memory space to store compression state.
+ Use LZ4_sizeofState() to know how much memory must be allocated,
+ and then, provide it as 'void* state' to compression functions.
+*/
+int LZ4_sizeofState(void);
+int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_decompress_fast (const char* source, char* dest, int outputSize);
/*
LZ4_decompress_fast() :
- outputSize : is the original (uncompressed) size
+ originalSize : is the original and therefore uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
- If the source stream is malformed, the function will stop decoding and return a negative result.
- note : This function is a bit faster than LZ4_decompress_safe()
- This function never writes outside of output buffers, and never read before input buffer, but may read beyond input buffer (since it doesn't know its size) in case of malicious data packet.
- Use this function preferably into a trusted environment (data to decode comes from a trusted source).
- Destination buffer must be already allocated. Its size must be a minimum of 'outputSize' bytes.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
+ Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
+ note : This function fully respect memory boundaries for properly formed compressed data.
+ It is a bit faster than LZ4_decompress_safe().
+ However, it does not provide any protection against intentionally modified data stream (malicious input).
+ Use this function in trusted environment only (data to decode comes from a trusted source).
*/
+int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
-int LZ4_decompress_safe_partial (const char* source, char* dest, int inputSize, int targetOutputSize, int maxOutputSize);
/*
LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'inputSize' at position 'source'
- into output buffer 'dest' of size 'maxOutputSize'.
- The function stops decompressing operation as soon as 'targetOutputSize' has been reached,
+ This function decompress a compressed block of size 'compressedSize' at position 'source'
+ into destination buffer 'dest' of size 'maxDecompressedSize'.
+ The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
- return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize)
+ return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
- If the source stream is malformed, the function will stop decoding and return a negative result.
+ If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/
+int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
-int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int outputSize);
+/***********************************************
+ Experimental Streaming Compression Functions
+***********************************************/
+#define LZ4_STREAMSIZE_U32 ((1 << (LZ4_MEMORY_USAGE-2)) + 8)
+#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U32 * sizeof(unsigned int))
/*
-*_withPrefix64k() :
- These decoding functions work the same as their "normal name" versions,
- but will potentially use up to 64KB of data in front of 'char* dest'.
- These functions are used for decoding inter-dependant blocks.
-*/
+ * LZ4_stream_t
+ * information structure to track an LZ4 stream.
+ * important : init this structure content before first use !
+ */
+typedef struct { unsigned int table[LZ4_STREAMSIZE_U32]; } LZ4_stream_t;
+
+/*
+ * LZ4_resetStream
+ * Use this function to init an allocated LZ4_stream_t structure
+ */
+void LZ4_resetStream (LZ4_stream_t* LZ4_streamPtr);
+
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStream will allocate and initialize an LZ4_stream_t structure
+ * LZ4_freeStream releases its memory.
+ */
+LZ4_stream_t* LZ4_createStream(void);
+int LZ4_freeStream (LZ4_stream_t* LZ4_stream);
+
+/*
+ * LZ4_loadDict
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Loading a size of 0 is allowed.
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_loadDict (LZ4_stream_t* LZ4_stream, const char* dictionary, int dictSize);
+
+/*
+ * LZ4_compress_continue
+ * Compress data block 'source', using blocks compressed before as dictionary to improve compression ratio
+ * Previous data blocks are assumed to still be present at their previous location.
+ */
+int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
+
+/*
+ * LZ4_compress_limitedOutput_continue
+ * Same as before, but also specify a maximum target compressed size (maxOutputSize)
+ * If objective cannot be met, compression exits, and returns a zero.
+ */
+int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/*
+ * LZ4_saveDict
+ * If previously compressed data block is not guaranteed to remain available at its memory location
+ * save it into a safer place (char* safeBuffer)
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ * dictionary is immediately usable, you can therefore call again LZ4_compress_continue()
+ * Return : dictionary size in bytes, or 0 if error
+ * Note : any dictSize > 64 KB will be interpreted as 64KB.
+ */
+int LZ4_saveDict (LZ4_stream_t* LZ4_stream, char* safeBuffer, int dictSize);
+
+
+/************************************************
+ Experimental Streaming Decompression Functions
+************************************************/
+
+#define LZ4_STREAMDECODESIZE_U32 4
+#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U32 * sizeof(unsigned int))
+/*
+ * LZ4_streamDecode_t
+ * information structure to track an LZ4 stream.
+ * important : init this structure content using LZ4_setStreamDecode or memset() before first use !
+ */
+typedef struct { unsigned int table[LZ4_STREAMDECODESIZE_U32]; } LZ4_streamDecode_t;
+/*
+ * LZ4_setStreamDecode
+ * Use this function to instruct where to find the dictionary.
+ * This function can be used to specify a static dictionary,
+ * or to instruct where to find some previously decoded data saved into a different memory space.
+ * Setting a size of 0 is allowed (same effect as no dictionary).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-//****************************
-// Obsolete Functions
-//****************************
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
+ * LZ4_freeStreamDecode releases its memory.
+ */
+LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+
+/*
+*_continue() :
+ These decoding functions allow decompression of multiple blocks in "streaming" mode.
+ Previously decoded blocks must still be available at the memory position where they were decoded.
+ If it's not possible, save the relevant part of decoded data into a safe buffer,
+ and indicate where its new address using LZ4_setStreamDecode()
+*/
+int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-static inline int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-static inline int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
/*
-These functions are deprecated and should no longer be used.
-They are provided here for compatibility with existing user programs.
+Advanced decoding functions :
+*_usingDict() :
+ These decoding functions work the same as
+ a combination of LZ4_setDictDecode() followed by LZ4_decompress_x_continue()
+ They don't use nor update an LZ4_streamDecode_t structure.
*/
+int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
+int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
+/**************************************
+ Obsolete Functions
+**************************************/
+/*
+Obsolete decompression functions
+These function names are deprecated and should no longer be used.
+They are only provided here for compatibility with older user programs.
+- LZ4_uncompress is the same as LZ4_decompress_fast
+- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
+These function prototypes are now disabled; uncomment them if you really need them.
+It is highly recommended to stop using these functions and migrated to newer ones */
+/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
+/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+
+/*
+ * If you prefer dynamic allocation methods,
+ * LZ4_createStreamDecode()
+ * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure.
+ * LZ4_free just frees it.
+ */
+/* void* LZ4_createStreamDecode(void); */
+/*int LZ4_free (void* LZ4_stream); yes, it's the same one as for compression */
+
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+void* LZ4_create (const char* inputBuffer);
+int LZ4_sizeofStreamState(void);
+int LZ4_resetStreamState(void* state, const char* inputBuffer);
+char* LZ4_slideInputBuffer (void* state);
+
+/* Obsolete streaming decoding functions */
+int LZ4_decompress_safe_withPrefix64k (const char* source, char* dest, int compressedSize, int maxOutputSize);
+int LZ4_decompress_fast_withPrefix64k (const char* source, char* dest, int originalSize);
+
#if defined (__cplusplus)
}
[36/50] [abbrv] git commit: HADOOP-11175. Fix several issues of
hadoop security configuration in user doc. Contributed by Yi Liu.
Posted by st...@apache.org.
HADOOP-11175. Fix several issues of hadoop security configuration in user doc. Contributed by Yi Liu.
(cherry picked from commit 1123a06e3658f26228f1eedaed796aca8efc7bad)
(cherry picked from commit 5af4c021cc3558de0ac8610170a1eb749e0171da)
(cherry picked from commit cf7eaf28e98226dc846d4910c84e408de0a9a4a1)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a5e0d42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a5e0d42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a5e0d42
Branch: refs/heads/champlain
Commit: 5a5e0d4202b0c433d2d25168435f9e289f2c6b40
Parents: 8901984
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 11:10:23 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 11:14:39 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/site/apt/SecureMode.apt.vm | 20 ++++++++------------
2 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a5e0d42/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 69b7622..7f41469 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -618,6 +618,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11117 UGI HadoopLoginModule doesn't catch & wrap all
kerberos-related exceptions (stevel)
+ HADOOP-11175. Fix several issues of hadoop security configuration in user
+ doc. (Yi Liu via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a5e0d42/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
index f42db2d..54473e6 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
@@ -402,8 +402,8 @@ Configuration for <<<conf/core-site.xml>>>
| <<<dfs.namenode.kerberos.principal>>> | nn/_HOST@REALM.TLD | |
| | | Kerberos principal name for the NameNode. |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.kerberos.https.principal>>> | host/_HOST@REALM.TLD | |
-| | | HTTPS Kerberos principal name for the NameNode. |
+| <<<dfs.namenode.kerberos.internal.spnego.principal>>> | HTTP/_HOST@REALM.TLD | |
+| | | HTTP Kerberos principal name for the NameNode. |
*-------------------------+-------------------------+------------------------+
Configuration for <<<conf/hdfs-site.xml>>>
@@ -416,16 +416,16 @@ Configuration for <<<conf/hdfs-site.xml>>>
*-------------------------+-------------------------+------------------------+
| <<<dfs.namenode.secondary.https-port>>> | <50470> | |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.secondary.keytab.file>>> | | |
+| <<<dfs.secondary.namenode.keytab.file>>> | | |
| | </etc/security/keytab/sn.service.keytab> | |
-| | | Kerberos keytab file for the NameNode. |
+| | | Kerberos keytab file for the Secondary NameNode. |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.secondary.kerberos.principal>>> | sn/_HOST@REALM.TLD | |
+| <<<dfs.secondary.namenode.kerberos.principal>>> | sn/_HOST@REALM.TLD | |
| | | Kerberos principal name for the Secondary NameNode. |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.secondary.kerberos.https.principal>>> | | |
-| | host/_HOST@REALM.TLD | |
-| | | HTTPS Kerberos principal name for the Secondary NameNode. |
+| <<<dfs.secondary.namenode.kerberos.internal.spnego.principal>>> | | |
+| | HTTP/_HOST@REALM.TLD | |
+| | | HTTP Kerberos principal name for the Secondary NameNode. |
*-------------------------+-------------------------+------------------------+
Configuration for <<<conf/hdfs-site.xml>>>
@@ -454,10 +454,6 @@ Configuration for <<<conf/hdfs-site.xml>>>
| <<<dfs.datanode.kerberos.principal>>> | dn/_HOST@REALM.TLD | |
| | | Kerberos principal name for the DataNode. |
*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.kerberos.https.principal>>> | | |
-| | host/_HOST@REALM.TLD | |
-| | | HTTPS Kerberos principal name for the DataNode. |
-*-------------------------+-------------------------+------------------------+
| <<<dfs.encrypt.data.transfer>>> | <false> | |
| | | set to <<<true>>> when using data encryption |
*-------------------------+-------------------------+------------------------+
[02/50] [abbrv] git commit: HADOOP-11048. user/custom LogManager
fails to load if the client classloader is enabled. Contributed by Sangjin
Lee (cherry picked from commit f154ebe8c44e41edc443198a14e0491604cc613f)
Posted by st...@apache.org.
HADOOP-11048. user/custom LogManager fails to load if the client classloader is enabled. Contributed by Sangjin Lee
(cherry picked from commit f154ebe8c44e41edc443198a14e0491604cc613f)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23f6657
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23f6657
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23f6657
Branch: refs/heads/champlain
Commit: a23f665707a3edc9c67a3a703170aebbf4d10180
Parents: a805d52
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Sep 26 19:49:23 2014 +0000
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:47:09 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/RunJar.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23f6657/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0a079b5..b57ae81 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -558,6 +558,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11179. Java untar should handle the case that the file entry comes
without its parent directory entry. (Craig Welch via zjshen)
+ HADOOP-11048. user/custom LogManager fails to load if the client
+ classloader is enabled (Sangjin Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23f6657/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 75b43b6..4b26b76 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -176,7 +176,7 @@ public class RunJar {
}
mainClassName = mainClassName.replaceAll("/", ".");
- File tmpDir = new File(new Configuration().get("hadoop.tmp.dir"));
+ File tmpDir = new File(System.getProperty("java.io.tmpdir"));
ensureDirectory(tmpDir);
final File workDir;
@@ -185,7 +185,7 @@ public class RunJar {
} catch (IOException ioe) {
// If user has insufficient perms to write to tmpDir, default
// "Permission denied" message doesn't specify a filename.
- System.err.println("Error creating temp dir in hadoop.tmp.dir "
+ System.err.println("Error creating temp dir in java.io.tmpdir "
+ tmpDir + " due to " + ioe.getMessage());
System.exit(-1);
return;
[32/50] [abbrv] git commit: HDFS-7203. Concurrent appendi to the same
file can cause data corruption. Contributed by Kihwal Lee. (cherry picked
from commit 853cb704edf54207313c0e70c9c375212d288b60)
Posted by st...@apache.org.
HDFS-7203. Concurrent appendi to the same file can cause data
corruption. Contributed by Kihwal Lee.
(cherry picked from commit 853cb704edf54207313c0e70c9c375212d288b60)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3c0ba0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3c0ba0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3c0ba0
Branch: refs/heads/champlain
Commit: ef3c0ba0945518e9bdbe50e01d1538bf70b23fca
Parents: 66d7fb2
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Oct 8 15:10:03 2014 -0500
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:17:28 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 9 ++-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 14 ++--
.../org/apache/hadoop/hdfs/TestFileAppend3.java | 83 ++++++++++++++++++++
3 files changed, 94 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3c0ba0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 723aafc..a30f3b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -529,9 +529,6 @@ Release 2.6.0 - UNRELEASED
HDFS-7148. TestEncryptionZones#testIsEncryptedMethod fails on branch-2
after archival storage merge. (wang)
- HDFS-7172. Test data files may be checked out of git with incorrect line
- endings, causing test failures in TestHDFSCLI. (Chris Nauroth via wheat9)
-
HDFS-7178. Additional unit test for replica write with full disk.
(Arpit Agarwal)
@@ -553,6 +550,12 @@ Release 2.6.0 - UNRELEASED
HDFS-7176. The namenode usage message doesn't include "-rollingupgrade
started" (cmccabe)
+ HDFS-7172. Test data files may be checked out of git with incorrect line
+ endings, causing test failures in TestHDFSCLI. (Chris Nauroth via wheat9)
+
+ HDFS-7203. Concurrent appending to the same file can cause data corruption
+ (kihwal)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3c0ba0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index c975ad5..66a4d0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1651,7 +1651,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
return null;
}
- return callAppend(stat, src, buffersize, progress);
+ return callAppend(src, buffersize, progress);
}
return null;
}
@@ -1723,7 +1723,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/** Method to get stream returned by append call */
- private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
+ private DFSOutputStream callAppend(String src,
int buffersize, Progressable progress) throws IOException {
LocatedBlock lastBlock = null;
try {
@@ -1737,8 +1737,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
}
+ HdfsFileStatus newStat = getFileInfo(src);
return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
- lastBlock, stat, dfsClientConf.createChecksum());
+ lastBlock, newStat, dfsClientConf.createChecksum());
}
/**
@@ -1762,12 +1763,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
private DFSOutputStream append(String src, int buffersize, Progressable progress)
throws IOException {
checkOpen();
- HdfsFileStatus stat = getFileInfo(src);
- if (stat == null) { // No file found
- throw new FileNotFoundException("failed to append to non-existent file "
- + src + " on client " + clientName);
- }
- final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
+ final DFSOutputStream result = callAppend(src, buffersize, progress);
beginFileLease(result.getFileId(), result);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3c0ba0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
index 66a04e7..d5de0ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
@@ -25,15 +25,23 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import org.mockito.invocation.InvocationOnMock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import org.mockito.stubbing.Answer;
+
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSClientAdapter;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -361,4 +369,79 @@ public class TestFileAppend3 {
AppendTestUtil.checkFullFile(fs, p, fileLen,
fileContents, "Failed to append to a partial chunk");
}
+
+ // Do small appends.
+ void doSmallAppends(Path file, DistributedFileSystem fs, int iterations)
+ throws IOException {
+ for (int i = 0; i < iterations; i++) {
+ FSDataOutputStream stm;
+ try {
+ stm = fs.append(file);
+ } catch (IOException e) {
+ // If another thread is already appending, skip this time.
+ continue;
+ }
+ // Failure in write or close will be terminal.
+ AppendTestUtil.write(stm, 0, 123);
+ stm.close();
+ }
+ }
+
+
+ @Test
+ public void testSmallAppendRace() throws Exception {
+ final Path file = new Path("/testSmallAppendRace");
+ final String fName = file.toUri().getPath();
+
+ // Create the file and write a small amount of data.
+ FSDataOutputStream stm = fs.create(file);
+ AppendTestUtil.write(stm, 0, 123);
+ stm.close();
+
+ // Introduce a delay between getFileInfo and calling append() against NN.
+ final DFSClient client = DFSClientAdapter.getDFSClient(fs);
+ DFSClient spyClient = spy(client);
+ when(spyClient.getFileInfo(fName)).thenAnswer(new Answer<HdfsFileStatus>() {
+ @Override
+ public HdfsFileStatus answer(InvocationOnMock invocation){
+ try {
+ HdfsFileStatus stat = client.getFileInfo(fName);
+ Thread.sleep(100);
+ return stat;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ });
+
+ DFSClientAdapter.setDFSClient(fs, spyClient);
+
+ // Create two threads for doing appends to the same file.
+ Thread worker1 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ doSmallAppends(file, fs, 20);
+ } catch (IOException e) {
+ }
+ }
+ };
+
+ Thread worker2 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ doSmallAppends(file, fs, 20);
+ } catch (IOException e) {
+ }
+ }
+ };
+
+ worker1.start();
+ worker2.start();
+
+ // append will fail when the file size crosses the checksum chunk boundary,
+ // if append was called with a stale file stat.
+ doSmallAppends(file, fs, 20);
+ }
}
[18/50] [abbrv] git commit: HDFS-6779. Add missing version subcommand
for hdfs. Contributed by Sasaki Toru.
Posted by st...@apache.org.
HDFS-6779. Add missing version subcommand for hdfs. Contributed by Sasaki Toru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0fdda81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0fdda81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0fdda81
Branch: refs/heads/champlain
Commit: a0fdda816d46f1f66547cb1858d14a58cfe5392a
Parents: 4ceea84
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Sep 29 18:10:47 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:25:40 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 5 ++++-
.../hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm | 6 ++++++
3 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0fdda81/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e702277..e719821 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -291,6 +291,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7124. Remove EncryptionZoneManager.NULL_EZ. (clamb via wang)
+ HDFS-6779. Add missing version subcommand for hdfs.
+ (Sasaki Toru via wheat9)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0fdda81/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 9c1a8d1..6184c01 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -70,6 +70,7 @@ function print_usage(){
echo " cacheadmin configure the HDFS cache"
echo " crypto configure HDFS encryption zones"
echo " storagepolicies get all the existing block storage policies"
+ echo " version print the version"
echo ""
echo "Most commands print help when invoked w/o parameters."
}
@@ -117,7 +118,7 @@ if [ "$COMMAND" == "nfs3" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_PRIVILEGED_NFS
if [ -n "$HADOOP_PRIVILEGED_NFS_PID_DIR" ]; then
HADOOP_PID_DIR=$HADOOP_PRIVILEGED_NFS_PID_DIR
fi
-
+
if [ -n "$HADOOP_PRIVILEGED_NFS_LOG_DIR" ]; then
HADOOP_LOG_DIR=$HADOOP_PRIVILEGED_NFS_LOG_DIR
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
@@ -200,6 +201,8 @@ elif [ "$COMMAND" = "cacheadmin" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
elif [ "$COMMAND" = "crypto" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
+elif [ "$COMMAND" = "version" ] ; then
+ CLASS=org.apache.hadoop.util.VersionInfo
else
CLASS="$COMMAND"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0fdda81/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 1e73377..7085110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -114,6 +114,12 @@ HDFS Commands Guide
| -racks | Print out network topology for data-node locations.
*------------------------+---------------------------------------------+
+** <<<version>>>
+
+ Prints the version.
+
+ Usage: <<<hdfs version>>>
+
* Administration Commands
Commands useful for administrators of a hadoop cluster.
[48/50] [abbrv] git commit: Monarch HADOOP-918: automatically expand
page blobs when they become full
Posted by st...@apache.org.
Monarch HADOOP-918: automatically expand page blobs when they become full
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7f34c78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7f34c78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7f34c78
Branch: refs/heads/champlain
Commit: b7f34c78fde5d7b4519564e4b932be654883a508
Parents: 30c8199
Author: Eric Hanson <ehans at microsoft dot com>
Authored: Fri Oct 10 10:53:33 2014 -0700
Committer: Eric Hanson <ehans at microsoft dot com>
Committed: Fri Oct 10 10:53:33 2014 -0700
----------------------------------------------------------------------
.../fs/azurenative/PageBlobOutputStream.java | 98 ++++++++++++++++++--
.../TestReadAndSeekPageBlobAfterWrite.java | 31 +++++++
2 files changed, 123 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7f34c78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/azurenative/PageBlobOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/azurenative/PageBlobOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/azurenative/PageBlobOutputStream.java
index 6c8a855..d62e606 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/azurenative/PageBlobOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/azurenative/PageBlobOutputStream.java
@@ -16,6 +16,8 @@ import org.apache.hadoop.conf.Configuration;
import com.microsoft.windowsazure.storage.OperationContext;
import com.microsoft.windowsazure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions;
+import com.microsoft.windowsazure.storage.blob.CloudBlob;
+import com.microsoft.windowsazure.storage.blob.CloudPageBlob;
import static org.apache.hadoop.fs.azurenative.PageBlobFormatHelpers.*;
@@ -62,6 +64,11 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
private volatile IOException lastError;
/**
+ * Current size of the page blob in bytes. It may be extended if the file
+ * gets full.
+ */
+ private long currentBlobSize;
+ /**
* The current byte offset we're at in the blob (how many bytes we've
* uploaded to the server).
*/
@@ -93,13 +100,23 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
public static final Log LOG = LogFactory.getLog(AzureNativeFileSystemStore.class);
- // Set the minimum page blob file size to 128MB, which is >> the default block size of 32MB.
- // This default block size is often used as the hbase.regionserver.hlog.blocksize.
+ // Set the minimum page blob file size to 128MB, which is >> the default
+ // block size of 32MB. This default block size is often used as the
+ // hbase.regionserver.hlog.blocksize.
// The goal is to have a safe minimum size for HBase log files to allow them
- // to be filled and rolled without exceeding the minimum size. A larger size can be
- // used by setting the fs.azure.page.blob.size configuration variable.
+ // to be filled and rolled without exceeding the minimum size. A larger size
+ // can be used by setting the fs.azure.page.blob.size configuration variable.
public static final long PAGE_BLOB_MIN_SIZE = 128L * 1024L * 1024L;
+ // The default and minimum amount to extend a page blob by if it starts
+ // to get full.
+ public static final long
+ PAGE_BLOB_DEFAULT_EXTENSION_SIZE = 128L * 1024L * 1024L;
+
+ // The configured page blob extension size (either the default, or if greater,
+ // the value configured in fs.azure.page.blob.extension.size
+ private long configuredPageBlobExtensionSize;
+
/**
* Constructs an output stream over the given page blob.
*
@@ -122,8 +139,6 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
this.ioThreadPool = new ThreadPoolExecutor(1, 1, 2, TimeUnit.SECONDS,
ioQueue);
-
-
// Make page blob files have a size that is the greater of a
// minimum size, or the value of fs.azure.page.blob.size from configuration.
long pageBlobConfigSize = conf.getLong("fs.azure.page.blob.size", 0);
@@ -136,6 +151,21 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
pageBlobSize += PAGE_SIZE - pageBlobSize % PAGE_SIZE;
}
blob.create(pageBlobSize, new BlobRequestOptions(), opContext);
+ currentBlobSize = pageBlobSize;
+
+ // Set the page blob extension size. It must be a minimum of the default
+ // value.
+ configuredPageBlobExtensionSize =
+ conf.getLong("fs.azure.page.blob.extension.size", 0);
+ if (configuredPageBlobExtensionSize < PAGE_BLOB_DEFAULT_EXTENSION_SIZE) {
+ configuredPageBlobExtensionSize = PAGE_BLOB_DEFAULT_EXTENSION_SIZE;
+ }
+
+ // make sure it is a multiple of the page size
+ if (configuredPageBlobExtensionSize % PAGE_SIZE != 0) {
+ configuredPageBlobExtensionSize +=
+ PAGE_SIZE - configuredPageBlobExtensionSize % PAGE_SIZE;
+ }
}
private void checkStreamState() throws IOException {
@@ -287,6 +317,12 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
// It wasn't a partial page, we won't need to rewrite it.
previousLastPageDataWritten = new byte[0];
}
+
+ // Extend the file if we need more room in the file. This typically takes
+ // less than 200 milliseconds if it has to actually be done,
+ // so it is okay to include it in a write and won't cause a long pause.
+ // Other writes can be queued behind this write in any case.
+ conditionalExtendFile();
}
/**
@@ -326,6 +362,56 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
}
/**
+ * Extend the page blob file if we are close to the end.
+ */
+ private void conditionalExtendFile() {
+
+ // maximum allowed size of an Azure page blob (1 terabyte)
+ final long MAX_PAGE_BLOB_SIZE = 1024L * 1024L * 1024L * 1024L;
+
+ // If blob is already at the maximum size, then don't try to extend it.
+ if (currentBlobSize == MAX_PAGE_BLOB_SIZE) {
+ return;
+ }
+
+ // If we are within the maximum write size of the end of the file,
+ if (currentBlobSize - currentBlobOffset <= MAX_RAW_BYTES_PER_REQUEST) {
+
+ // Extend the file. Retry up to 3 times with back-off.
+ CloudPageBlob cloudPageBlob = (CloudPageBlob) blob.getBlob();
+ long newSize = currentBlobSize + configuredPageBlobExtensionSize;
+
+ // Make sure we don't exceed maximum blob size.
+ if (newSize > MAX_PAGE_BLOB_SIZE) {
+ newSize = MAX_PAGE_BLOB_SIZE;
+ }
+ final int MAX_RETRIES = 3;
+ int retries = 1;
+ boolean resizeDone = false;
+ while(!resizeDone && retries <= MAX_RETRIES) {
+ try {
+ cloudPageBlob.resize(newSize);
+ resizeDone = true;
+ currentBlobSize = newSize;
+ } catch (StorageException e) {
+ LOG.warn("Failed to extend size of " + cloudPageBlob.getUri());
+ try {
+
+ // sleep 2, 8, 18 seconds for up to 3 retries
+ Thread.sleep(2000 * retries * retries);
+ } catch (InterruptedException e1) {
+
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ }
+ } finally {
+ retries++;
+ }
+ }
+ }
+ }
+
+ /**
* Flushes this output stream and forces any buffered output bytes to be
* written out. If any data remains in the buffer it is committed to the
* service. Data is queued for writing but not forced out to the service
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7f34c78/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/azurenative/TestReadAndSeekPageBlobAfterWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/azurenative/TestReadAndSeekPageBlobAfterWrite.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/azurenative/TestReadAndSeekPageBlobAfterWrite.java
index 6226655..01271e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/azurenative/TestReadAndSeekPageBlobAfterWrite.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/azurenative/TestReadAndSeekPageBlobAfterWrite.java
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureException;
@@ -334,4 +335,34 @@ public class TestReadAndSeekPageBlobAfterWrite {
writeAndReadOneFile(numWrites, recordSize, syncInterval);
}
}
+
+ // Write to a file repeatedly to verify that it extends.
+ // The page blob file should start out at 128MB and finish at 256MB.
+ @Test(timeout=300000)
+ public void testFileSizeExtension() throws IOException {
+ final int writeSize = 1024 * 1024;
+ final int numWrites = 129;
+ final byte dataByte = 5;
+ byte[] data = new byte[writeSize];
+ Arrays.fill(data, dataByte);
+ FSDataOutputStream output = fs.create(PATH);
+ try {
+ for (int i = 0; i < numWrites; i++) {
+ output.write(data);
+ output.hflush();
+ LOG.debug("total writes = " + (i + 1));
+ }
+ } finally {
+ output.close();
+ }
+
+ // Show that we wrote more than the default page blob file size.
+ assertTrue(numWrites * writeSize > PageBlobOutputStream.PAGE_BLOB_MIN_SIZE);
+
+ // Verify we can list the new size. That will prove we expanded the file.
+ FileStatus[] status = fs.listStatus(PATH);
+ assertTrue(status[0].getLen() == numWrites * writeSize);
+ LOG.debug("Total bytes written to " + PATH + " = " + status[0].getLen());
+ fs.delete(PATH, false);
+ }
}
[08/50] [abbrv] git commit: HDFS-6519. Document oiv_legacy command
(Akira AJISAKA via aw)
Posted by st...@apache.org.
HDFS-6519. Document oiv_legacy command (Akira AJISAKA via aw)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b9e1a6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b9e1a6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b9e1a6c
Branch: refs/heads/champlain
Commit: 2b9e1a6c8c4613351a800680f850fc23eab2afe8
Parents: 215e132
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Sep 29 08:51:55 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:56:49 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++
.../src/main/resources/hdfs-default.xml | 11 ++++
.../src/site/apt/HdfsImageViewer.apt.vm | 59 +++++++++++++++++++-
3 files changed, 73 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b9e1a6c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0a0f50e..5afe95d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -280,6 +280,10 @@ Release 2.6.0 - UNRELEASED
HDFS-7156. Update fsck documentation. (Masahiro Yamaguch via shv)
+ HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal)
+
+ HDFS-6519. Document oiv_legacy command (Akira AJISAKA via aw)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b9e1a6c/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 6396227..3da0cdb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2150,4 +2150,15 @@
</description>
</property>
+<property>
+ <name>dfs.namenode.legacy-oiv-image.dir</name>
+ <value></value>
+ <description>Determines where to save the namespace in the old fsimage format
+ during checkpointing by standby NameNode or SecondaryNameNode. Users can
+ dump the contents of the old format fsimage by oiv_legacy command. If
+ the value is not specified, old format fsimage will not be saved in
+ checkpoint.
+ </description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b9e1a6c/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsImageViewer.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsImageViewer.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsImageViewer.apt.vm
index 9a9946e..3b84226 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsImageViewer.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsImageViewer.apt.vm
@@ -28,7 +28,7 @@ Offline Image Viewer Guide
namespace. The tool is able to process very large image files relatively
quickly. The tool handles the layout formats that were included with Hadoop
versions 2.4 and up. If you want to handle older layout formats, you can
- use the Offline Image Viewer of Hadoop 2.3.
+ use the Offline Image Viewer of Hadoop 2.3 or {{oiv_legacy Command}}.
If the tool is not able to process an image file, it will exit cleanly.
The Offline Image Viewer does not require a Hadoop cluster to be running;
it is entirely offline in its operation.
@@ -188,3 +188,60 @@ Offline Image Viewer Guide
about the hdfs namespace. This information can then be used to explore
file system usage patterns or find specific files that match arbitrary
criteria, along with other types of namespace analysis.
+
+* oiv_legacy Command
+
+ Due to the internal layout changes introduced by the ProtocolBuffer-based
+ fsimage ({{{https://issues.apache.org/jira/browse/HDFS-5698}HDFS-5698}}),
+ OfflineImageViewer consumes excessive amount of memory and loses some
+ functions such as Indented and Delimited processor. If you want to process
+ without large amount of memory or use these processors, you can use
+ <<<oiv_legacy>>> command (same as <<<oiv>>> in Hadoop 2.3).
+
+** Usage
+
+ 1. Set <<<dfs.namenode.legacy-oiv-image.dir>>> to an appropriate directory
+ to make standby NameNode or SecondaryNameNode save its namespace in the
+ old fsimage format during checkpointing.
+
+ 2. Use <<<oiv_legacy>>> command to the old format fsimage.
+
+----
+ bash$ bin/hdfs oiv_legacy -i fsimage_old -o output
+----
+
+** Options
+
+*-----------------------:-----------------------------------+
+| <<Flag>> | <<Description>> |
+*-----------------------:-----------------------------------+
+| <<<-i>>>\|<<<--inputFile>>> <input file> | Specify the input fsimage file to
+| | process. Required.
+*-----------------------:-----------------------------------+
+| <<<-o>>>\|<<<--outputFile>>> <output file> | Specify the output filename, if
+| | the specified output processor generates one. If the
+| | specified file already exists, it is silently
+| | overwritten. Required.
+*-----------------------:-----------------------------------+
+| <<<-p>>>\|<<<--processor>>> <processor> | Specify the image processor to
+| | apply against the image file. Valid options are
+| | Ls (default), XML, Delimited, Indented, and
+| | FileDistribution.
+*-----------------------:-----------------------------------+
+| <<<-skipBlocks>>> | Do not enumerate individual blocks within files. This
+| | may save processing time and outfile file space on
+| | namespaces with very large files. The Ls processor
+| | reads the blocks to correctly determine file sizes
+| | and ignores this option.
+*-----------------------:-----------------------------------+
+| <<<-printToScreen>>> | Pipe output of processor to console as well as
+| | specified file. On extremely large namespaces, this
+| | may increase processing time by an order of
+| | magnitude.
+*-----------------------:-----------------------------------+
+| <<<-delimiter>>> <arg>| When used in conjunction with the Delimited
+| | processor, replaces the default tab delimiter with
+| | the string specified by <arg>.
+*-----------------------:-----------------------------------+
+| <<<-h>>>\|<<<--help>>>| Display the tool usage and help information and exit.
+*-----------------------:-----------------------------------+
[21/50] [abbrv] git commit: HADOOP-11117 UGI HadoopLoginModule
doesn't catch & wrap all kerberos-related exceptions (stevel)
Posted by st...@apache.org.
HADOOP-11117 UGI HadoopLoginModule doesn't catch & wrap all kerberos-related exceptions (stevel)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4040cbf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4040cbf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4040cbf7
Branch: refs/heads/champlain
Commit: 4040cbf783a74e6f89c65b4e7d5be74d0b7d93a4
Parents: a3605e8
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 30 17:28:56 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:32:56 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++++
.../java/org/apache/hadoop/security/User.java | 3 ++-
.../hadoop/security/UserGroupInformation.java | 18 ++++++++++++++++--
.../hadoop/security/TestUserGroupInformation.java | 3 ++-
4 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4040cbf7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d415d03..ce33214 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -589,6 +589,11 @@ Release 2.6.0 - UNRELEASED
HADOOP-11154. Update BUILDING.txt to state that CMake 3.0 or newer is
required on Mac. (cnauroth)
+ HADOOP-11145. TestFairCallQueue fails. (Akira AJISAKA via cnauroth)
+
+ HADOOP-11117 UGI HadoopLoginModule doesn't catch & wrap all
+ kerberos-related exceptions (stevel)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4040cbf7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/User.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/User.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/User.java
index 8d9b28b..236e962 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/User.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/User.java
@@ -47,7 +47,8 @@ class User implements Principal {
try {
shortName = new HadoopKerberosName(name).getShortName();
} catch (IOException ioe) {
- throw new IllegalArgumentException("Illegal principal name " + name, ioe);
+ throw new IllegalArgumentException("Illegal principal name " + name
+ +": " + ioe.toString(), ioe);
}
fullName = name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4040cbf7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 4f117fd..fbefdb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -176,7 +176,21 @@ public class UserGroupInformation {
}
// if we found the user, add our principal
if (user != null) {
- subject.getPrincipals().add(new User(user.getName()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using user: \"" + user + "\" with name " + user.getName());
+ }
+
+ User userEntry = null;
+ try {
+ userEntry = new User(user.getName());
+ } catch (Exception e) {
+ throw (LoginException)(new LoginException(e.toString()).initCause(e));
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User entry: \"" + userEntry.toString() + "\"" );
+ }
+
+ subject.getPrincipals().add(userEntry);
return true;
}
LOG.error("Can't find user in " + subject);
@@ -919,7 +933,7 @@ public class UserGroupInformation {
metrics.loginFailure.add(Time.now() - start);
}
throw new IOException("Login failure for " + user + " from keytab " +
- path, le);
+ path+ ": " + le, le);
}
LOG.info("Login successful for user " + keytabPrincipal
+ " using keytab file " + keytabFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4040cbf7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 23e89d8..f1fba03 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -339,7 +339,8 @@ public class TestUserGroupInformation {
} catch (IllegalArgumentException e) {
String expect = (userName == null || userName.isEmpty())
? "Null user" : "Illegal principal name "+userName;
- assertEquals(expect, e.getMessage());
+ assertTrue("Did not find "+ expect + " in " + e,
+ e.toString().contains(expect));
}
}
[40/50] [abbrv] git commit: YARN-2598 GHS should show N/A instead of
null for the inaccessible information (Zhijie Shen via mayank) (cherry picked
from commit c1a86d105e1c21b9eb1217b001e2df51fae068da)
Posted by st...@apache.org.
YARN-2598 GHS should show N/A instead of null for the inaccessible information (Zhijie Shen via mayank)
(cherry picked from commit c1a86d105e1c21b9eb1217b001e2df51fae068da)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b45cbc89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b45cbc89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b45cbc89
Branch: refs/heads/champlain
Commit: b45cbc89613c6cac0f58acf9c52bc696ccbe5b01
Parents: 01ea43f
Author: Mayank Bansal <ma...@apache.org>
Authored: Thu Oct 9 13:41:38 2014 -0700
Committer: Jian He <jh...@hortonworks.com>
Committed: Thu Oct 9 20:36:57 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
...pplicationHistoryManagerOnTimelineStore.java | 20 +++++++++++++++++
...pplicationHistoryManagerOnTimelineStore.java | 23 +++++++++++---------
3 files changed, 36 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45cbc89/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 18976ed..07c0514 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -507,6 +507,9 @@ Release 2.6.0 - UNRELEASED
YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
+ YARN-2598 GHS should show N/A instead of null for the inaccessible information
+ (Zhijie Shen via mayank)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45cbc89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 5381bd6..e1a4d1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -59,10 +59,15 @@ import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import com.google.common.annotations.VisibleForTesting;
+
public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
implements
ApplicationHistoryManager {
+ @VisibleForTesting
+ static final String UNAVAILABLE = "N/A";
+
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
@@ -508,6 +513,21 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
app.appReport.setCurrentApplicationAttemptId(
ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1));
}
+ if (app.appReport.getHost() == null) {
+ app.appReport.setHost(UNAVAILABLE);
+ }
+ if (app.appReport.getRpcPort() < 0) {
+ app.appReport.setRpcPort(-1);
+ }
+ if (app.appReport.getTrackingUrl() == null) {
+ app.appReport.setTrackingUrl(UNAVAILABLE);
+ }
+ if (app.appReport.getOriginalTrackingUrl() == null) {
+ app.appReport.setOriginalTrackingUrl(UNAVAILABLE);
+ }
+ if (app.appReport.getDiagnostics() == null) {
+ app.appReport.setDiagnostics("");
+ }
return app;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45cbc89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index f6c1481..311626e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -178,16 +178,19 @@ public class TestApplicationHistoryManagerOnTimelineStore {
callerUGI.getShortUserName().equals("user3")))) {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
app.getCurrentApplicationAttemptId());
- Assert.assertEquals(null, app.getHost());
+ Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
+ app.getHost());
Assert.assertEquals(-1, app.getRpcPort());
- Assert.assertEquals(null, app.getTrackingUrl());
- Assert.assertEquals(null, app.getOriginalTrackingUrl());
- Assert.assertEquals(null, app.getDiagnostics());
+ Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
+ app.getTrackingUrl());
+ Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
+ app.getOriginalTrackingUrl());
+ Assert.assertEquals("", app.getDiagnostics());
} else {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
app.getCurrentApplicationAttemptId());
Assert.assertEquals("test host", app.getHost());
- Assert.assertEquals(-100, app.getRpcPort());
+ Assert.assertEquals(100, app.getRpcPort());
Assert.assertEquals("test tracking url", app.getTrackingUrl());
Assert.assertEquals("test original tracking url",
app.getOriginalTrackingUrl());
@@ -236,7 +239,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
appAttempt.getAMContainerId());
Assert.assertEquals("test host", appAttempt.getHost());
- Assert.assertEquals(-100, appAttempt.getRpcPort());
+ Assert.assertEquals(100, appAttempt.getRpcPort());
Assert.assertEquals("test tracking url", appAttempt.getTrackingUrl());
Assert.assertEquals("test original tracking url",
appAttempt.getOriginalTrackingUrl());
@@ -282,7 +285,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
Assert.assertEquals(Resource.newInstance(-1, -1),
container.getAllocatedResource());
- Assert.assertEquals(NodeId.newInstance("test host", -100),
+ Assert.assertEquals(NodeId.newInstance("test host", 100),
container.getAssignedNode());
Assert.assertEquals(Priority.UNDEFINED, container.getPriority());
Assert
@@ -290,7 +293,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState());
Assert.assertEquals(-1, container.getContainerExitStatus());
Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
- "test host:-100/container_0_0001_01_000001/"
+ "test host:100/container_0_0001_01_000001/"
+ "container_0_0001_01_000001/user1", container.getLogUrl());
}
@@ -471,7 +474,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
"test original tracking url");
eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, "test host");
- eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, -100);
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, 100);
eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
ContainerId.newInstance(appAttemptId, 1));
tEvent.setEventInfo(eventInfo);
@@ -509,7 +512,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, -1);
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
"test host");
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, -100);
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, 100);
entityInfo
.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, -1);
entity.setOtherInfo(entityInfo);
[39/50] [abbrv] git commit: YARN-2629. Made the distributed shell use
the domain-based timeline ACLs. Contributed by Zhijie Shen.
Posted by st...@apache.org.
YARN-2629. Made the distributed shell use the domain-based timeline ACLs. Contributed by Zhijie Shen.
(cherry picked from commit 1d4612f5ad9678c952b416e798dccd20c88f96ef)
(cherry picked from commit 4d170d3db6bd0b07e5e49b6b41a5a60d59466e66)
Conflicts:
hadoop-yarn-project/CHANGES.txt
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01ea43fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01ea43fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01ea43fd
Branch: refs/heads/champlain
Commit: 01ea43fd7c1762482fbff78b70cd934a98f412ad
Parents: b08182c
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu Oct 9 12:59:47 2014 -0700
Committer: Jian He <jh...@hortonworks.com>
Committed: Thu Oct 9 20:36:37 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 120 ++++++++++++-------
.../applications/distributedshell/Client.java | 75 +++++++++++-
.../distributedshell/DSConstants.java | 5 +
.../distributedshell/TestDistributedShell.java | 52 +++++++-
.../client/api/impl/TimelineClientImpl.java | 3 -
6 files changed, 205 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 46778a8..18976ed 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -274,6 +274,9 @@ Release 2.6.0 - UNRELEASED
YARN-2562. Changed ContainerId#toString() to be more readable. (Tsuyoshi
OZAWA via jianhe)
+ YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
+ (zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index df9f34b..e6ded00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@@ -86,6 +87,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -246,6 +248,9 @@ public class ApplicationMaster {
// File length needed for local resource
private long shellScriptPathLen = 0;
+ // Timeline domain ID
+ private String domainId = null;
+
// Hardcoded path to shell script in launch container's local env
private static final String ExecShellStringPath = Client.SCRIPT_PATH + ".sh";
private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
@@ -465,7 +470,9 @@ public class ApplicationMaster {
shellScriptPathLen = Long.valueOf(envs
.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
-
+ if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) {
+ domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN);
+ }
if (!scriptPath.isEmpty()
&& (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
LOG.error("Illegal values in env for shell script path" + ", path="
@@ -515,13 +522,6 @@ public class ApplicationMaster {
@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException {
LOG.info("Starting ApplicationMaster");
- try {
- publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
- DSEvent.DS_APP_ATTEMPT_START);
- } catch (Exception e) {
- LOG.error("App Attempt start event could not be published for "
- + appAttemptID.toString(), e);
- }
// Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
// are marked as LimitedPrivate
@@ -548,6 +548,9 @@ public class ApplicationMaster {
UserGroupInformation.createRemoteUser(appSubmitterUserName);
appSubmitterUgi.addCredentials(credentials);
+ publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+ DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+
AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
amRMClient.init(conf);
@@ -612,13 +615,9 @@ public class ApplicationMaster {
amRMClient.addContainerRequest(containerAsk);
}
numRequestedContainers.set(numTotalContainers);
- try {
- publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
- DSEvent.DS_APP_ATTEMPT_END);
- } catch (Exception e) {
- LOG.error("App Attempt start event could not be published for "
- + appAttemptID.toString(), e);
- }
+
+ publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+ DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
}
@VisibleForTesting
@@ -724,12 +723,8 @@ public class ApplicationMaster {
LOG.info("Container completed successfully." + ", containerId="
+ containerStatus.getContainerId());
}
- try {
- publishContainerEndEvent(timelineClient, containerStatus);
- } catch (Exception e) {
- LOG.error("Container start event could not be published for "
- + containerStatus.getContainerId().toString(), e);
- }
+ publishContainerEndEvent(
+ timelineClient, containerStatus, domainId, appSubmitterUgi);
}
// ask for more containers if any failed
@@ -844,13 +839,9 @@ public class ApplicationMaster {
if (container != null) {
applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
}
- try {
- ApplicationMaster.publishContainerStartEvent(
- applicationMaster.timelineClient, container);
- } catch (Exception e) {
- LOG.error("Container start event could not be published for "
- + container.getId().toString(), e);
- }
+ ApplicationMaster.publishContainerStartEvent(
+ applicationMaster.timelineClient, container,
+ applicationMaster.domainId, applicationMaster.appSubmitterUgi);
}
@Override
@@ -1050,13 +1041,14 @@ public class ApplicationMaster {
}
}
- private static void publishContainerStartEvent(TimelineClient timelineClient,
- Container container) throws IOException, YarnException {
- TimelineEntity entity = new TimelineEntity();
+ private static void publishContainerStartEvent(
+ final TimelineClient timelineClient, Container container, String domainId,
+ UserGroupInformation ugi) {
+ final TimelineEntity entity = new TimelineEntity();
entity.setEntityId(container.getId().toString());
entity.setEntityType(DSEntity.DS_CONTAINER.toString());
- entity.addPrimaryFilter("user",
- UserGroupInformation.getCurrentUser().getShortUserName());
+ entity.setDomainId(domainId);
+ entity.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType(DSEvent.DS_CONTAINER_START.toString());
@@ -1064,16 +1056,28 @@ public class ApplicationMaster {
event.addEventInfo("Resources", container.getResource().toString());
entity.addEvent(event);
- timelineClient.putEntities(entity);
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
+ @Override
+ public TimelinePutResponse run() throws Exception {
+ return timelineClient.putEntities(entity);
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("Container start event could not be published for "
+ + container.getId().toString(),
+ e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ }
}
- private static void publishContainerEndEvent(TimelineClient timelineClient,
- ContainerStatus container) throws IOException, YarnException {
- TimelineEntity entity = new TimelineEntity();
+ private static void publishContainerEndEvent(
+ final TimelineClient timelineClient, ContainerStatus container,
+ String domainId, UserGroupInformation ugi) {
+ final TimelineEntity entity = new TimelineEntity();
entity.setEntityId(container.getContainerId().toString());
entity.setEntityType(DSEntity.DS_CONTAINER.toString());
- entity.addPrimaryFilter("user",
- UserGroupInformation.getCurrentUser().getShortUserName());
+ entity.setDomainId(domainId);
+ entity.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType(DSEvent.DS_CONTAINER_END.toString());
@@ -1081,22 +1085,46 @@ public class ApplicationMaster {
event.addEventInfo("Exit Status", container.getExitStatus());
entity.addEvent(event);
- timelineClient.putEntities(entity);
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
+ @Override
+ public TimelinePutResponse run() throws Exception {
+ return timelineClient.putEntities(entity);
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("Container end event could not be published for "
+ + container.getContainerId().toString(),
+ e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ }
}
private static void publishApplicationAttemptEvent(
- TimelineClient timelineClient, String appAttemptId, DSEvent appEvent)
- throws IOException, YarnException {
- TimelineEntity entity = new TimelineEntity();
+ final TimelineClient timelineClient, String appAttemptId,
+ DSEvent appEvent, String domainId, UserGroupInformation ugi) {
+ final TimelineEntity entity = new TimelineEntity();
entity.setEntityId(appAttemptId);
entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
- entity.addPrimaryFilter("user",
- UserGroupInformation.getCurrentUser().getShortUserName());
+ entity.setDomainId(domainId);
+ entity.addPrimaryFilter("user", ugi.getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setEventType(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
entity.addEvent(event);
- timelineClient.putEntities(entity);
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
+ @Override
+ public TimelinePutResponse run() throws Exception {
+ return timelineClient.putEntities(entity);
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("App Attempt "
+ + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ + " event could not be published for "
+ + appAttemptId.toString(),
+ e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 5287b8a..fe41727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -71,12 +71,15 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
/**
* Client for Distributed Shell application submission to YARN.
@@ -175,7 +178,19 @@ public class Client {
private long logRollingIntervalSeconds = 0;
// Debug flag
- boolean debugFlag = false;
+ boolean debugFlag = false;
+
+ // Timeline domain ID
+ private String domainId = null;
+
+ // Flag to indicate whether to create the domain of the given ID
+ private boolean toCreateDomain = false;
+
+ // Timeline domain reader access control
+ private String viewACLs = null;
+
+ // Timeline domain writer access control
+ private String modifyACLs = null;
// Command line options
private Options opts;
@@ -274,6 +289,14 @@ public class Client {
opts.addOption("logRollingIntervalSeconds", true, "how often the "
+ "logAggregationSerivce uploads container logs in seconds.");
opts.addOption("debug", false, "Dump out debug information");
+ opts.addOption("domain", true, "ID of the timeline domain where the "
+ + "timeline entities will be put");
+ opts.addOption("view_acls", true, "Users and groups that allowed to "
+ + "view the timeline entities in the given domain");
+ opts.addOption("modify_acls", true, "Users and groups that allowed to "
+ + "modify the timeline entities in the given domain");
+ opts.addOption("create", false, "Flag to indicate whether to create the "
+ + "domain specified with -domain.");
opts.addOption("help", false, "Print usage");
opts.addOption("label_expression", true, "Set label expression will be used by this application");
}
@@ -416,6 +439,18 @@ public class Client {
"logRollingIntervalSeconds", "-1"));
log4jPropFile = cliParser.getOptionValue("log_properties", "");
+ // Get timeline domain options
+ if (cliParser.hasOption("domain")) {
+ domainId = cliParser.getOptionValue("domain");
+ toCreateDomain = cliParser.hasOption("create");
+ if (cliParser.hasOption("view_acls")) {
+ viewACLs = cliParser.getOptionValue("view_acls");
+ }
+ if (cliParser.hasOption("modify_acls")) {
+ modifyACLs = cliParser.getOptionValue("modify_acls");
+ }
+ }
+
return true;
}
@@ -463,6 +498,10 @@ public class Client {
}
}
+ if (domainId != null && domainId.length() > 0 && toCreateDomain) {
+ prepareTimelineDomain();
+ }
+
// Get a new application id
YarnClientApplication app = yarnClient.createApplication();
GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
@@ -579,6 +618,9 @@ public class Client {
env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp));
env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen));
+ if (domainId != null && domainId.length() > 0) {
+ env.put(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN, domainId);
+ }
// Add AppMaster.jar location to classpath
// At some point we should not be required to add
@@ -820,4 +862,35 @@ public class Client {
scFileStatus.getLen(), scFileStatus.getModificationTime());
localResources.put(fileDstPath, scRsrc);
}
+
+ private void prepareTimelineDomain() {
+ TimelineClient timelineClient = null;
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+ timelineClient = TimelineClient.createTimelineClient();
+ timelineClient.init(conf);
+ timelineClient.start();
+ } else {
+ LOG.warn("Cannot put the domain " + domainId +
+ " because the timeline service is not enabled");
+ return;
+ }
+ try {
+ //TODO: we need to check and combine the existing timeline domain ACLs,
+ //but let's do it once we have client java library to query domains.
+ TimelineDomain domain = new TimelineDomain();
+ domain.setId(domainId);
+ domain.setReaders(
+ viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
+ domain.setWriters(
+ modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
+ timelineClient.putDomain(domain);
+ LOG.info("Put the timeline domain: " +
+ TimelineUtils.dumpTimelineRecordtoJSON(domain));
+ } catch (Exception e) {
+ LOG.error("Error when putting the timeline domain", e);
+ } finally {
+ timelineClient.stop();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java
index 5912f14..fbaf2d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DSConstants.java
@@ -44,4 +44,9 @@ public class DSConstants {
* Used to validate the local resource.
*/
public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN";
+
+ /**
+ * Environment key name denoting the timeline domain ID.
+ */
+ public static final String DISTRIBUTEDSHELLTIMELINEDOMAIN = "DISTRIBUTEDSHELLTIMELINEDOMAIN";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 6dff94c..2414d4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -29,10 +29,10 @@ import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -43,6 +43,7 @@ import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManag
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -129,8 +131,16 @@ public class TestDistributedShell {
}
@Test(timeout=90000)
- public void testDSShell() throws Exception {
+ public void testDSShellWithDomain() throws Exception {
+ testDSShell(true);
+ }
+ @Test(timeout=90000)
+ public void testDSShellWithoutDomain() throws Exception {
+ testDSShell(false);
+ }
+
+ public void testDSShell(boolean haveDomain) throws Exception {
String[] args = {
"--jar",
APPMASTER_JAR,
@@ -147,6 +157,20 @@ public class TestDistributedShell {
"--container_vcores",
"1"
};
+ if (haveDomain) {
+ String[] domainArgs = {
+ "--domain",
+ "TEST_DOMAIN",
+ "--view_acls",
+ "reader_user reader_group",
+ "--modify_acls",
+ "writer_user writer_group",
+ "--create"
+ };
+ List<String> argsList = new ArrayList<String>(Arrays.asList(args));
+ argsList.addAll(Arrays.asList(domainArgs));
+ args = argsList.toArray(new String[argsList.size()]);
+ }
LOG.info("Initializing DS Client");
final Client client = new Client(new Configuration(yarnCluster.getConfig()));
@@ -198,7 +222,15 @@ public class TestDistributedShell {
t.join();
LOG.info("Client run completed. Result=" + result);
Assert.assertTrue(result.get());
-
+
+ TimelineDomain domain = null;
+ if (haveDomain) {
+ domain = yarnCluster.getApplicationHistoryServer()
+ .getTimelineStore().getDomain("TEST_DOMAIN");
+ Assert.assertNotNull(domain);
+ Assert.assertEquals("reader_user reader_group", domain.getReaders());
+ Assert.assertEquals("writer_user writer_group", domain.getWriters());
+ }
TimelineEntities entitiesAttempts = yarnCluster
.getApplicationHistoryServer()
.getTimelineStore()
@@ -210,6 +242,13 @@ public class TestDistributedShell {
.size());
Assert.assertEquals(entitiesAttempts.getEntities().get(0).getEntityType()
.toString(), ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString());
+ if (haveDomain) {
+ Assert.assertEquals(domain.getId(),
+ entitiesAttempts.getEntities().get(0).getDomainId());
+ } else {
+ Assert.assertEquals("DEFAULT",
+ entitiesAttempts.getEntities().get(0).getDomainId());
+ }
TimelineEntities entities = yarnCluster
.getApplicationHistoryServer()
.getTimelineStore()
@@ -219,6 +258,13 @@ public class TestDistributedShell {
Assert.assertEquals(2, entities.getEntities().size());
Assert.assertEquals(entities.getEntities().get(0).getEntityType()
.toString(), ApplicationMaster.DSEntity.DS_CONTAINER.toString());
+ if (haveDomain) {
+ Assert.assertEquals(domain.getId(),
+ entities.getEntities().get(0).getDomainId());
+ } else {
+ Assert.assertEquals("DEFAULT",
+ entities.getEntities().get(0).getDomainId());
+ }
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01ea43fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 13ce074..fbddd14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -210,9 +210,6 @@ public class TimelineClientImpl extends TimelineClient {
@VisibleForTesting
public ClientResponse doPostingObject(Object object, String path) {
WebResource webResource = client.resource(resURI);
- if (path != null) {
- webResource.path(path);
- }
if (path == null) {
return webResource.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
[20/50] [abbrv] git commit: HADOOP-11113. Namenode not able to
reconnect to KMS after KMS restart. (Arun Suresh via wang)
Posted by st...@apache.org.
HADOOP-11113. Namenode not able to reconnect to KMS after KMS restart. (Arun Suresh via wang)
(cherry picked from commit e25a25c5343c889d8c9e45b65082ddb55cf36d52)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3605e89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3605e89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3605e89
Branch: refs/heads/champlain
Commit: a3605e8925851a03c30c953241c27873895d99a4
Parents: e327b49
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Sep 30 16:46:58 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:27:01 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../crypto/key/kms/KMSClientProvider.java | 10 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 19 ++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 118 +++++++++++++++++--
4 files changed, 132 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3605e89/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0cbc53c..d415d03 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -445,6 +445,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11130. NFS updateMaps OS check is reversed (brandonli)
+ HADOOP-11113. Namenode not able to reconnect to KMS after KMS restart.
+ (Arun Suresh via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3605e89/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index e3aa1dc..bc57343 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -415,7 +415,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return conn;
}
- private static <T> T call(HttpURLConnection conn, Map jsonOutput,
+ private <T> T call(HttpURLConnection conn, Map jsonOutput,
int expectedResponse, Class<T> klass)
throws IOException {
T ret = null;
@@ -427,6 +427,14 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
conn.getInputStream().close();
throw ex;
}
+ if (conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
+ // Ideally, this should happen only when there is an Authentication
+ // failure. Unfortunately, the AuthenticationFilter returns 403 when it
+ // cannot authenticate (Since a 401 requires Server to send
+ // WWW-Authenticate header as well)..
+ KMSClientProvider.this.authToken =
+ new DelegationTokenAuthenticatedURL.Token();
+ }
HttpExceptionUtils.validateResponse(conn, expectedResponse);
if (APPLICATION_JSON_MIME.equalsIgnoreCase(conn.getContentType())
&& klass != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3605e89/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index 697d7ec..51cc026 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -43,12 +43,12 @@ import java.util.UUID;
public class MiniKMS {
- private static Server createJettyServer(String keyStore, String password) {
+ private static Server createJettyServer(String keyStore, String password, int inPort) {
try {
boolean ssl = keyStore != null;
InetAddress localhost = InetAddress.getByName("localhost");
String host = "localhost";
- ServerSocket ss = new ServerSocket(0, 50, localhost);
+ ServerSocket ss = new ServerSocket((inPort < 0) ? 0 : inPort, 50, localhost);
int port = ss.getLocalPort();
ss.close();
Server server = new Server(0);
@@ -91,6 +91,7 @@ public class MiniKMS {
private String log4jConfFile;
private File keyStoreFile;
private String keyStorePassword;
+ private int inPort = -1;
public Builder() {
kmsConfDir = new File("target/test-classes").getAbsoluteFile();
@@ -111,6 +112,12 @@ public class MiniKMS {
return this;
}
+ public Builder setPort(int port) {
+ Preconditions.checkArgument(port > 0, "input port must be greater than 0");
+ this.inPort = port;
+ return this;
+ }
+
public Builder setSslConf(File keyStoreFile, String keyStorePassword) {
Preconditions.checkNotNull(keyStoreFile, "keystore file is NULL");
Preconditions.checkNotNull(keyStorePassword, "keystore password is NULL");
@@ -126,7 +133,7 @@ public class MiniKMS {
"KMS conf dir does not exist");
return new MiniKMS(kmsConfDir.getAbsolutePath(), log4jConfFile,
(keyStoreFile != null) ? keyStoreFile.getAbsolutePath() : null,
- keyStorePassword);
+ keyStorePassword, inPort);
}
}
@@ -135,14 +142,16 @@ public class MiniKMS {
private String keyStore;
private String keyStorePassword;
private Server jetty;
+ private int inPort;
private URL kmsURL;
public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
- String password) {
+ String password, int inPort) {
this.kmsConfDir = kmsConfDir;
this.log4jConfFile = log4ConfFile;
this.keyStore = keyStore;
this.keyStorePassword = password;
+ this.inPort = inPort;
}
public void start() throws Exception {
@@ -174,7 +183,7 @@ public class MiniKMS {
writer.close();
}
System.setProperty("log4j.configuration", log4jConfFile);
- jetty = createJettyServer(keyStore, keyStorePassword);
+ jetty = createJettyServer(keyStore, keyStorePassword, inPort);
// we need to do a special handling for MiniKMS to work when in a dir and
// when in a JAR in the classpath thanks to Jetty way of handling of webapps
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3605e89/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 9211417..4f106e6 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -89,7 +89,7 @@ public class TestKMS {
return file;
}
- public static abstract class KMSCallable implements Callable<Void> {
+ public static abstract class KMSCallable<T> implements Callable<T> {
private URL kmsUrl;
protected URL getKMSUrl() {
@@ -97,19 +97,27 @@ public class TestKMS {
}
}
- protected void runServer(String keystore, String password, File confDir,
- KMSCallable callable) throws Exception {
+ protected <T> T runServer(String keystore, String password, File confDir,
+ KMSCallable<T> callable) throws Exception {
+ return runServer(-1, keystore, password, confDir, callable);
+ }
+
+ protected <T> T runServer(int port, String keystore, String password, File confDir,
+ KMSCallable<T> callable) throws Exception {
MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder().setKmsConfDir(confDir)
.setLog4jConfFile("log4j.properties");
if (keystore != null) {
miniKMSBuilder.setSslConf(new File(keystore), password);
}
+ if (port > 0) {
+ miniKMSBuilder.setPort(port);
+ }
MiniKMS miniKMS = miniKMSBuilder.build();
miniKMS.start();
try {
System.out.println("Test KMS running at: " + miniKMS.getKMSUrl());
callable.kmsUrl = miniKMS.getKMSUrl();
- callable.call();
+ return callable.call();
} finally {
miniKMS.stop();
}
@@ -284,7 +292,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(keystore, password, testDir, new KMSCallable() {
+ runServer(keystore, password, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
@@ -351,7 +359,7 @@ public class TestKMS {
conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k6.ALL", "*");
writeConf(confDir, conf);
- runServer(null, null, confDir, new KMSCallable() {
+ runServer(null, null, confDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
Date started = new Date();
@@ -616,7 +624,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
@@ -783,6 +791,92 @@ public class TestKMS {
}
@Test
+ public void testKMSRestart() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ final File testDir = getTestDir();
+ conf = createBaseKMSConf(testDir);
+ conf.set("hadoop.kms.authentication.kerberos.keytab",
+ keytab.getAbsolutePath());
+ conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
+ conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+
+ for (KMSACLs.Type type : KMSACLs.Type.values()) {
+ conf.set(type.getAclConfigKey(), type.toString());
+ }
+ conf.set(KMSACLs.Type.CREATE.getAclConfigKey(),
+ KMSACLs.Type.CREATE.toString() + ",SET_KEY_MATERIAL");
+
+ conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(),
+ KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL");
+
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k3.ALL", "*");
+
+ writeConf(testDir, conf);
+
+ KMSCallable<KeyProvider> c =
+ new KMSCallable<KeyProvider>() {
+ @Override
+ public KeyProvider call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ final URI uri = createKMSUri(getKMSUrl());
+
+ final KeyProvider kp =
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<KeyProvider>() {
+ @Override
+ public KeyProvider run() throws Exception {
+ KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ kp.createKey("k1", new byte[16],
+ new KeyProvider.Options(conf));
+ return kp;
+ }
+ });
+ return kp;
+ }
+ };
+
+ final KeyProvider retKp =
+ runServer(null, null, testDir, c);
+
+ // Restart server (using the same port)
+ runServer(c.getKMSUrl().getPort(), null, null, testDir,
+ new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ retKp.createKey("k2", new byte[16],
+ new KeyProvider.Options(conf));
+ Assert.fail("Should fail first time !!");
+ } catch (IOException e) {
+ String message = e.getMessage();
+ Assert.assertTrue("Should be a 403 error : " + message,
+ message.contains("403"));
+ }
+ retKp.createKey("k2", new byte[16],
+ new KeyProvider.Options(conf));
+ retKp.createKey("k3", new byte[16],
+ new KeyProvider.Options(conf));
+ return null;
+ }
+ });
+ return null;
+ }
+ });
+ }
+
+ @Test
public void testACLs() throws Exception {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
@@ -809,7 +903,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
@@ -1117,7 +1211,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
@@ -1201,7 +1295,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
@@ -1326,7 +1420,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
@@ -1398,7 +1492,7 @@ public class TestKMS {
writeConf(testDir, conf);
- runServer(null, null, testDir, new KMSCallable() {
+ runServer(null, null, testDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
final Configuration conf = new Configuration();
[45/50] [abbrv] git commit: MAPREDUCE-6123.
TestCombineFileInputFormat incorrectly starts 2 MiniDFSCluster instances.
Contributed by Chris Nauroth.
Posted by st...@apache.org.
MAPREDUCE-6123. TestCombineFileInputFormat incorrectly starts 2 MiniDFSCluster instances. Contributed by Chris Nauroth.
(cherry picked from commit 777877d079ecb025c293deac741800800e413179)
(cherry picked from commit f6278cbac50522302ee7ad3cbbe21d0aeefa76d8)
(cherry picked from commit 308e3368533990d3cde74c30cbc30be861c8809b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5794c706
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5794c706
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5794c706
Branch: refs/heads/champlain
Commit: 5794c706db0b2b9f253218aa91c27f76405c083a
Parents: 75e4230
Author: cnauroth <cn...@apache.org>
Authored: Thu Oct 9 22:35:58 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Oct 9 22:55:12 2014 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java | 4 ----
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5794c706/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a75d8a8..57f3e17 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -195,6 +195,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6122. TestLineRecordReader may fail due to test data files checked
out of git with incorrect line endings. (cnauroth)
+ MAPREDUCE-6123. TestCombineFileInputFormat incorrectly starts 2
+ MiniDFSCluster instances. (cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5794c706/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
index 4290914..db51ec6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
@@ -1286,10 +1286,6 @@ public class TestCombineFileInputFormat extends TestCase {
.build();
dfs.waitActive();
- dfs = new MiniDFSCluster.Builder(conf).racks(rack1).hosts(hosts1)
- .build();
- dfs.waitActive();
-
FileSystem fileSys = dfs.getFileSystem();
// Set up the following directory structure:
[16/50] [abbrv] git commit: HADOOP-11153. Make number of KMS threads
configurable. (wang)
Posted by st...@apache.org.
HADOOP-11153. Make number of KMS threads configurable. (wang)
(cherry picked from commit 64aef189656919327b11fe5e90ea680ca7ba83df)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b45b45b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b45b45b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b45b45b2
Branch: refs/heads/champlain
Commit: b45b45b21486c45e334f80ac0f0f537d70ec7fbd
Parents: ba0c76c
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 29 15:02:29 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:23:51 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh | 4 ++++
.../hadoop-kms/src/main/libexec/kms-config.sh | 7 +++++++
hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh | 1 +
hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml | 1 +
.../hadoop-kms/src/main/tomcat/ssl-server.xml | 2 +-
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 2 ++
7 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index cabaadc..e174903 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -230,6 +230,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10731. Remove @date JavaDoc comment in ProgramDriver class (Henry
Saputra via aw)
+ HADOOP-11153. Make number of KMS threads configurable. (wang)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index 1bbec7f..88a2b86 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -36,6 +36,10 @@
#
# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+# The maximum number of Tomcat handler threads
+#
+# export KMS_MAX_THREADS=1000
+
# The location of the SSL keystore if using SSL
#
# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
index 3e20399..dc603d4 100644
--- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
@@ -136,6 +136,13 @@ else
print "Using KMS_ADMIN_PORT: ${KMS_ADMIN_PORT}"
fi
+if [ "${KMS_MAX_THREADS}" = "" ]; then
+ export KMS_MAX_THREADS=1000
+ print "Setting KMS_MAX_THREADS: ${KMS_MAX_THREADS}"
+else
+ print "Using KMS_MAX_THREADS: ${KMS_MAX_THREADS}"
+fi
+
if [ "${KMS_SSL_KEYSTORE_FILE}" = "" ]; then
export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
print "Setting KMS_SSL_KEYSTORE_FILE: ${KMS_SSL_KEYSTORE_FILE}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
index 588c2c8..b43225f 100644
--- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
@@ -44,6 +44,7 @@ catalina_opts="${catalina_opts} -Dkms.log.dir=${KMS_LOG}";
catalina_opts="${catalina_opts} -Dkms.temp.dir=${KMS_TEMP}";
catalina_opts="${catalina_opts} -Dkms.admin.port=${KMS_ADMIN_PORT}";
catalina_opts="${catalina_opts} -Dkms.http.port=${KMS_HTTP_PORT}";
+catalina_opts="${catalina_opts} -Dkms.max.threads=${KMS_MAX_THREADS}";
catalina_opts="${catalina_opts} -Dkms.ssl.keystore.file=${KMS_SSL_KEYSTORE_FILE}";
catalina_opts="${catalina_opts} -Dkms.ssl.keystore.pass=${KMS_SSL_KEYSTORE_PASS}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml b/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
index d070b58..516a6a9 100644
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/tomcat/server.xml
@@ -73,6 +73,7 @@
Define a non-SSL HTTP/1.1 Connector on port ${kms.http.port}
-->
<Connector port="${kms.http.port}" protocol="HTTP/1.1"
+ maxThreads="${kms.max.threads}"
connectionTimeout="20000"
redirectPort="8443"/>
<!-- A "Connector" using the shared thread pool-->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml b/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
index 3a464ca..6235b8b 100644
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
@@ -69,7 +69,7 @@
connector should be using the OpenSSL style configuration
described in the APR documentation -->
<Connector port="${kms.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
- maxThreads="150" scheme="https" secure="true"
+ maxThreads="${kms.max.threads}" scheme="https" secure="true"
clientAuth="false" sslProtocol="TLS"
keystoreFile="${kms.ssl.keystore.file}"
keystorePass="${kms.ssl.keystore.pass}"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b45b45b2/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 2e8405f..76eb1a6 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -152,6 +152,8 @@ hadoop-${project.version} $ sbin/kms.sh start
* KMS_ADMIN_PORT
+ * KMS_MAX_THREADS
+
* KMS_LOG
NOTE: You need to restart the KMS for the configuration changes to take
[03/50] [abbrv] git commit: HADOOP-10731. Remove @date JavaDoc
comment in ProgramDriver class (Henry Saputra via aw)
Posted by st...@apache.org.
HADOOP-10731. Remove @date JavaDoc comment in ProgramDriver class (Henry Saputra via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37969b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37969b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37969b6
Branch: refs/heads/champlain
Commit: d37969b61573b23eb1c5831d6a5fc4d5eb393bf8
Parents: a23f665
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 26 14:41:53 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 22:48:20 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/ProgramDriver.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37969b6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b57ae81..adc92c5d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -227,6 +227,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10954. Adding site documents of hadoop-tools (Masatake Iwasaki
via aw)
+ HADOOP-10731. Remove @date JavaDoc comment in ProgramDriver class (Henry
+ Saputra via aw)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37969b6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
index fb018cf..347e508 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
@@ -35,7 +35,6 @@ public class ProgramDriver {
/**
* A description of a program based on its class and a
* human-readable description.
- * @date april 2006
*/
Map<String, ProgramDescription> programs;
[28/50] [abbrv] git commit: HADOOP-11151. Automatically refresh auth
token and retry on auth failure. Contributed by Arun Suresh.
Posted by st...@apache.org.
HADOOP-11151. Automatically refresh auth token and retry on auth failure. Contributed by Arun Suresh.
(cherry picked from commit 2d8e6e2c4a52a4ba815b23d6d1ac21be4df23d9e)
(cherry picked from commit 9ebff016c2de3410622e6da3e9963adbaa894c1a)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4baf1310
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4baf1310
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4baf1310
Branch: refs/heads/champlain
Commit: 4baf13100b1f860836dc3a679734471d91189b2c
Parents: fa9e4e2
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 2 19:54:57 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:06:47 2014 -0700
----------------------------------------------------------------------
.../authentication/client/AuthenticatedURL.java | 5 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +
.../crypto/key/kms/KMSClientProvider.java | 44 ++++++-
.../src/main/resources/core-default.xml | 7 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 127 +++++++++++++++++--
5 files changed, 176 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4baf1310/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index cee951f..61c3c6d 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -250,7 +250,10 @@ public class AuthenticatedURL {
* @throws AuthenticationException if an authentication exception occurred.
*/
public static void extractToken(HttpURLConnection conn, Token token) throws IOException, AuthenticationException {
- if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
+ int respCode = conn.getResponseCode();
+ if (respCode == HttpURLConnection.HTTP_OK
+ || respCode == HttpURLConnection.HTTP_CREATED
+ || respCode == HttpURLConnection.HTTP_ACCEPTED) {
Map<String, List<String>> headers = conn.getHeaderFields();
List<String> cookies = headers.get("Set-Cookie");
if (cookies != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4baf1310/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ce33214..e7e972ad 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -448,6 +448,12 @@ Release 2.6.0 - UNRELEASED
HADOOP-11113. Namenode not able to reconnect to KMS after KMS restart.
(Arun Suresh via wang)
+ HADOOP-11160. Fix typo in nfs3 server duplicate entry reporting.
+ (Charles Lamb via wheat9)
+
+ HADOOP-11151. Automatically refresh auth token and retry on auth failure.
+ (Arun Suresh via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4baf1310/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index bc57343..5b7f109 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -76,6 +77,8 @@ import com.google.common.base.Preconditions;
public class KMSClientProvider extends KeyProvider implements CryptoExtension,
KeyProviderDelegationTokenExtension.DelegationTokenExtension {
+ private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed";
+
public static final String TOKEN_KIND = "kms-dt";
public static final String SCHEME_NAME = "kms";
@@ -97,6 +100,13 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
public static final String TIMEOUT_ATTR = CONFIG_PREFIX + "timeout";
public static final int DEFAULT_TIMEOUT = 60;
+ /* Number of times to retry authentication in the event of auth failure
+ * (normally happens due to stale authToken)
+ */
+ public static final String AUTH_RETRY = CONFIG_PREFIX
+ + "authentication.retry-count";
+ public static final int DEFAULT_AUTH_RETRY = 1;
+
private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
private class EncryptedQueueRefiller implements
@@ -238,6 +248,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private ConnectionConfigurator configurator;
private DelegationTokenAuthenticatedURL.Token authToken;
private UserGroupInformation loginUgi;
+ private final int authRetry;
@Override
public String toString() {
@@ -296,6 +307,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
int timeout = conf.getInt(TIMEOUT_ATTR, DEFAULT_TIMEOUT);
+ authRetry = conf.getInt(AUTH_RETRY, DEFAULT_AUTH_RETRY);
configurator = new TimeoutConnConfigurator(timeout, sslFactory);
encKeyVersionQueue =
new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>(
@@ -416,7 +428,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
private <T> T call(HttpURLConnection conn, Map jsonOutput,
- int expectedResponse, Class<T> klass)
+ int expectedResponse, Class<T> klass) throws IOException {
+ return call(conn, jsonOutput, expectedResponse, klass, authRetry);
+ }
+
+ private <T> T call(HttpURLConnection conn, Map jsonOutput,
+ int expectedResponse, Class<T> klass, int authRetryCount)
throws IOException {
T ret = null;
try {
@@ -427,13 +444,36 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
conn.getInputStream().close();
throw ex;
}
- if (conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN) {
+ if ((conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN
+ && conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED))
+ || conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
// Ideally, this should happen only when there is an Authentication
// failure. Unfortunately, the AuthenticationFilter returns 403 when it
// cannot authenticate (Since a 401 requires Server to send
// WWW-Authenticate header as well)..
KMSClientProvider.this.authToken =
new DelegationTokenAuthenticatedURL.Token();
+ KMSClientProvider.this.loginUgi =
+ UserGroupInformation.getCurrentUser();
+ if (authRetryCount > 0) {
+ String contentType = conn.getRequestProperty(CONTENT_TYPE);
+ String requestMethod = conn.getRequestMethod();
+ URL url = conn.getURL();
+ conn = createConnection(url, requestMethod);
+ conn.setRequestProperty(CONTENT_TYPE, contentType);
+ return call(conn, jsonOutput, expectedResponse, klass,
+ authRetryCount - 1);
+ }
+ }
+ try {
+ AuthenticatedURL.extractToken(conn, authToken);
+ } catch (AuthenticationException e) {
+ // Ignore the AuthExceptions.. since we are just using the method to
+ // extract and set the authToken.. (Workaround till we actually fix
+ // AuthenticatedURL properly to set authToken post initialization)
+ } finally {
+ KMSClientProvider.this.loginUgi =
+ UserGroupInformation.getCurrentUser();
}
HttpExceptionUtils.validateResponse(conn, expectedResponse);
if (APPLICATION_JSON_MIME.equalsIgnoreCase(conn.getContentType())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4baf1310/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a85e2b9..a6b913a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1606,6 +1606,13 @@ for ldap providers in the same way as above does.
<!--- KMSClientProvider configurations -->
<property>
+ <name>hadoop.security.kms.client.authentication.retry-count</name>
+ <value>1</value>
+ <description>
+ Number of time to retry connecting to KMS on authentication failure
+ </description>
+</property>
+<property>
<name>hadoop.security.kms.client.encrypted.key.cache.size</name>
<value>500</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4baf1310/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 4f106e6..afa2d27 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,8 +32,10 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -49,6 +51,8 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@@ -791,12 +795,24 @@ public class TestKMS {
}
@Test
- public void testKMSRestart() throws Exception {
+ public void testKMSRestartKerberosAuth() throws Exception {
+ doKMSRestart(true);
+ }
+
+ @Test
+ public void testKMSRestartSimpleAuth() throws Exception {
+ doKMSRestart(false);
+ }
+
+ public void doKMSRestart(boolean useKrb) throws Exception {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
final File testDir = getTestDir();
conf = createBaseKMSConf(testDir);
+ if (useKrb) {
+ conf.set("hadoop.kms.authentication.type", "kerberos");
+ }
conf.set("hadoop.kms.authentication.kerberos.keytab",
keytab.getAbsolutePath());
conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
@@ -855,15 +871,6 @@ public class TestKMS {
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- try {
- retKp.createKey("k2", new byte[16],
- new KeyProvider.Options(conf));
- Assert.fail("Should fail first time !!");
- } catch (IOException e) {
- String message = e.getMessage();
- Assert.assertTrue("Should be a 403 error : " + message,
- message.contains("403"));
- }
retKp.createKey("k2", new byte[16],
new KeyProvider.Options(conf));
retKp.createKey("k3", new byte[16],
@@ -877,6 +884,106 @@ public class TestKMS {
}
@Test
+ public void testKMSAuthFailureRetry() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ final File testDir = getTestDir();
+ conf = createBaseKMSConf(testDir);
+ conf.set("hadoop.kms.authentication.kerberos.keytab",
+ keytab.getAbsolutePath());
+ conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
+ conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+
+ for (KMSACLs.Type type : KMSACLs.Type.values()) {
+ conf.set(type.getAclConfigKey(), type.toString());
+ }
+ conf.set(KMSACLs.Type.CREATE.getAclConfigKey(),
+ KMSACLs.Type.CREATE.toString() + ",SET_KEY_MATERIAL");
+
+ conf.set(KMSACLs.Type.ROLLOVER.getAclConfigKey(),
+ KMSACLs.Type.ROLLOVER.toString() + ",SET_KEY_MATERIAL");
+
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k0.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k1.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k2.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k3.ALL", "*");
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "k4.ALL", "*");
+
+ writeConf(testDir, conf);
+
+ runServer(null, null, testDir,
+ new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ final URI uri = createKMSUri(getKMSUrl());
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ kp.createKey("k1", new byte[16],
+ new KeyProvider.Options(conf));
+ makeAuthTokenStale(kp);
+ kp.createKey("k2", new byte[16],
+ new KeyProvider.Options(conf));
+ return null;
+ }
+ });
+ return null;
+ }
+ });
+
+ // Test retry count
+ runServer(null, null, testDir,
+ new KMSCallable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 128);
+ conf.setInt(KMSClientProvider.AUTH_RETRY, 0);
+ final URI uri = createKMSUri(getKMSUrl());
+ doAs("SET_KEY_MATERIAL",
+ new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ kp.createKey("k3", new byte[16],
+ new KeyProvider.Options(conf));
+ makeAuthTokenStale(kp);
+ try {
+ kp.createKey("k4", new byte[16],
+ new KeyProvider.Options(conf));
+ Assert.fail("Shoud fail since retry count == 0");
+ } catch (IOException e) {
+ Assert.assertTrue(
+ "HTTP exception must be a 403 : " + e.getMessage(), e
+ .getMessage().contains("403"));
+ }
+ return null;
+ }
+ });
+ return null;
+ }
+ });
+ }
+
+ private void makeAuthTokenStale(KMSClientProvider kp) throws Exception {
+ Field tokF = KMSClientProvider.class.getDeclaredField("authToken");
+ tokF.setAccessible(true);
+ DelegationTokenAuthenticatedURL.Token delToken =
+ (DelegationTokenAuthenticatedURL.Token) tokF.get(kp);
+ String oldTokStr = delToken.toString();
+ Method setM =
+ AuthenticatedURL.Token.class.getDeclaredMethod("set", String.class);
+ setM.setAccessible(true);
+ String newTokStr = oldTokStr.replaceAll("e=[^&]*", "e=1000");
+ setM.invoke(((AuthenticatedURL.Token)delToken), newTokStr);
+ }
+
+ @Test
public void testACLs() throws Exception {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
[23/50] [abbrv] git commit: Addendum patch for HDFS-7172. Contributed
by Chris Nauroth.
Posted by st...@apache.org.
Addendum patch for HDFS-7172. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09ac019f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09ac019f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09ac019f
Branch: refs/heads/champlain
Commit: 09ac019f3a97e8f60a42193c8cebe0a536d7be23
Parents: b16b1a3
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Oct 1 16:09:30 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Wed Oct 8 23:34:57 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09ac019f/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index b196293..35ccd57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -493,6 +493,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<excludes>
<exclude>CHANGES.txt</exclude>
<exclude>CHANGES.HDFS-1623.txt</exclude>
+ <exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
<exclude>src/main/conf/*</exclude>
<exclude>src/main/docs/**</exclude>
[31/50] [abbrv] git commit: HADOOP-11169. Fix
DelegationTokenAuthenticatedURL to pass the connection Configurator to the
authenticator. (Arun Suresh via wang)
Posted by st...@apache.org.
HADOOP-11169. Fix DelegationTokenAuthenticatedURL to pass the connection Configurator to the authenticator. (Arun Suresh via wang)
(cherry picked from commit b2f619752355d4ef6733935c020f57c8a26d82e1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66d7fb21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66d7fb21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66d7fb21
Branch: refs/heads/champlain
Commit: 66d7fb213fd514b7ad51575172fcbe298365ab6d
Parents: 7e1e11e
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Oct 7 14:46:59 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:15:21 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../delegation/web/DelegationTokenAuthenticatedURL.java | 12 +++++++++---
.../delegation/web/DelegationTokenAuthenticator.java | 4 +++-
.../apache/hadoop/crypto/key/kms/server/TestKMS.java | 5 +++++
4 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66d7fb21/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index afa18bc..e5db43c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -463,6 +463,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11168. Remove duplicated entry "dfs.webhdfs.enabled" in the useri
doc. (Yi Liu via wheat9)
+ HADOOP-11169. Fix DelegationTokenAuthenticatedURL to pass the connection
+ Configurator to the authenticator. (Arun Suresh via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66d7fb21/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 5aeb177..8c7cbdf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -117,9 +117,14 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
}
private static DelegationTokenAuthenticator
- obtainDelegationTokenAuthenticator(DelegationTokenAuthenticator dta) {
+ obtainDelegationTokenAuthenticator(DelegationTokenAuthenticator dta,
+ ConnectionConfigurator connConfigurator) {
try {
- return (dta != null) ? dta : DEFAULT_AUTHENTICATOR.newInstance();
+ if (dta == null) {
+ dta = DEFAULT_AUTHENTICATOR.newInstance();
+ dta.setConnectionConfigurator(connConfigurator);
+ }
+ return dta;
} catch (Exception ex) {
throw new IllegalArgumentException(ex);
}
@@ -169,7 +174,8 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
public DelegationTokenAuthenticatedURL(
DelegationTokenAuthenticator authenticator,
ConnectionConfigurator connConfigurator) {
- super(obtainDelegationTokenAuthenticator(authenticator), connConfigurator);
+ super(obtainDelegationTokenAuthenticator(authenticator, connConfigurator),
+ connConfigurator);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66d7fb21/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 7f22941..c614ee3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -95,6 +95,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
}
private Authenticator authenticator;
+ private ConnectionConfigurator connConfigurator;
public DelegationTokenAuthenticator(Authenticator authenticator) {
this.authenticator = authenticator;
@@ -103,6 +104,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
@Override
public void setConnectionConfigurator(ConnectionConfigurator configurator) {
authenticator.setConnectionConfigurator(configurator);
+ connConfigurator = configurator;
}
private boolean hasDelegationToken(URL url, AuthenticatedURL.Token token) {
@@ -215,7 +217,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
separator = "&";
}
url = new URL(sb.toString());
- AuthenticatedURL aUrl = new AuthenticatedURL(this);
+ AuthenticatedURL aUrl = new AuthenticatedURL(this, connConfigurator);
HttpURLConnection conn = aUrl.openConnection(url, token);
conn.setRequestMethod(operation.getHttpMethod());
HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66d7fb21/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index afa2d27..ad2f500 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -321,6 +322,10 @@ public class TestKMS {
KeyProvider kp = new KMSClientProvider(uri, conf);
// getKeys() empty
Assert.assertTrue(kp.getKeys().isEmpty());
+
+ Token<?>[] tokens = ((KMSClientProvider)kp).addDelegationTokens("myuser", new Credentials());
+ Assert.assertEquals(1, tokens.length);
+ Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
}
return null;
}