You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/12/09 00:59:04 UTC
[3/4] hive git commit: HIVE-11820 : export tables with size of >32MB
throws java.lang.IllegalArgumentException: Skip CRC is valid only with update
options (Takahiko Saito via Ashutosh Chauhan)
HIVE-11820 : export tables with size of >32MB throws java.lang.IllegalArgumentException: Skip CRC is valid only with update options (Takahiko Saito via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ea4073fc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ea4073fc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ea4073fc
Branch: refs/heads/branch-1
Commit: ea4073fc9041d829a06eff80a81e075d9dfba71d
Parents: 4cdd19a
Author: Takahiko Saito <ty...@gmail.com>
Authored: Wed Sep 16 15:34:00 2015 -0800
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Tue Dec 8 17:34:16 2015 -0600
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java | 5 ++++-
.../main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ea4073fc/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 32b072e..a79a36f 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -73,6 +73,8 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.distcp2.DistCp;
import org.apache.hadoop.tools.distcp2.DistCpOptions;
+import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute;
+
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.VersionInfo;
@@ -672,8 +674,9 @@ public class Hadoop20SShims extends HadoopShimsSecure {
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
- options.setSkipCRC(true);
options.setSyncFolder(true);
+ options.setSkipCRC(true);
+ options.preserve(FileAttribute.BLOCKSIZE);
try {
DistCp distcp = new DistCp(conf, options);
distcp.execute();
http://git-wip-us.apache.org/repos/asf/hive/blob/ea4073fc/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index cbac554..89789de 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.tools.DistCpOptions;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.test.MiniTezCluster;
@@ -1169,8 +1170,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
- options.setSkipCRC(true);
options.setSyncFolder(true);
+ options.setSkipCRC(true);
+ options.preserve(FileAttribute.BLOCKSIZE);
try {
DistCp distcp = new DistCp(conf, options);
distcp.execute();