You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/10/13 20:25:36 UTC
hadoop git commit: HADOOP-13024. Distcp with -delete feature on raw
data not implemented. Contributed by Mavin Martin.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8c721aa00 -> 0a85d0798
HADOOP-13024. Distcp with -delete feature on raw data not implemented. Contributed by Mavin Martin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a85d079
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a85d079
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a85d079
Branch: refs/heads/trunk
Commit: 0a85d079838f532a13ca237300386d1b3bc1b178
Parents: 8c721aa
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Oct 13 13:24:37 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Oct 13 13:24:54 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/tools/DistCpConstants.java | 12 +++++-
.../hadoop/tools/mapred/CopyCommitter.java | 5 ++-
.../hadoop/tools/TestDistCpWithRawXAttrs.java | 45 +++++++++-----------
.../hadoop/tools/util/DistCpTestUtils.java | 32 ++++++++------
4 files changed, 56 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index 96f364c..6171aa9 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.tools;
* limitations under the License.
*/
+import org.apache.hadoop.fs.Path;
+
/**
* Utility class to hold commonly used constants.
*/
@@ -125,9 +127,17 @@ public class DistCpConstants {
public static final int SPLIT_RATIO_DEFAULT = 2;
/**
+ * Constants for NONE file deletion
+ */
+ public static final String NONE_PATH_NAME = "/NONE";
+ public static final Path NONE_PATH = new Path(NONE_PATH_NAME);
+ public static final Path RAW_NONE_PATH = new Path(
+ DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME + NONE_PATH_NAME);
+
+ /**
* Value of reserved raw HDFS directory when copying raw.* xattrs.
*/
- static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
+ public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 6d2fef5..dd653b2 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -238,7 +238,10 @@ public class CopyCommitter extends FileOutputCommitter {
List<Path> targets = new ArrayList<Path>(1);
Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
targets.add(targetFinalPath);
- DistCpOptions options = new DistCpOptions(targets, new Path("/NONE"));
+ Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath)
+ .toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME)
+ ? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH;
+ DistCpOptions options = new DistCpOptions(targets, resultNonePath);
//
// Set up options to be the same from the CopyListing.buildListing's perspective,
// so to collect similar listings as when doing the copy
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
index 5aef51a..8adc2cf 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
@@ -82,14 +82,7 @@ public class TestDistCpWithRawXAttrs {
final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest";
doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true,
DistCpConstants.SUCCESS);
- doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px",
- false, true, DistCpConstants.SUCCESS);
- doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px",
- false, true, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px",
- false, true, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px",
- true, true, DistCpConstants.SUCCESS);
+ doTestStandardPreserveRawXAttrs("-px", true);
final Path savedWd = fs.getWorkingDirectory();
try {
fs.setWorkingDirectory(new Path("/.reserved/raw"));
@@ -103,27 +96,18 @@ public class TestDistCpWithRawXAttrs {
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
@Test
public void testPreserveRawXAttrs2() throws Exception {
- doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p",
- false, false, DistCpConstants.SUCCESS);
- doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p",
- false, false, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p",
- false, false, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p",
- true, false, DistCpConstants.SUCCESS);
+ doTestStandardPreserveRawXAttrs("-p", false);
}
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
@Test
public void testPreserveRawXAttrs3() throws Exception {
- doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null,
- false, false, DistCpConstants.SUCCESS);
- doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null,
- false, false, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null,
- false, false, DistCpConstants.INVALID_ARGUMENT);
- doTestPreserveRawXAttrs(rawSrcName, rawDestName, null,
- true, false, DistCpConstants.SUCCESS);
+ doTestStandardPreserveRawXAttrs(null, false);
+ }
+
+ @Test
+ public void testPreserveRawXAttrs4() throws Exception {
+ doTestStandardPreserveRawXAttrs("-update -delete", false);
}
private static Path[] pathnames = { new Path("dir1"),
@@ -145,6 +129,19 @@ public class TestDistCpWithRawXAttrs {
}
}
+ private void doTestStandardPreserveRawXAttrs(String options,
+ boolean expectUser)
+ throws Exception {
+ doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, options,
+ false, expectUser, DistCpConstants.SUCCESS);
+ doTestPreserveRawXAttrs(rootedSrcName, rawDestName, options,
+ false, expectUser, DistCpConstants.INVALID_ARGUMENT);
+ doTestPreserveRawXAttrs(rawSrcName, rootedDestName, options,
+ false, expectUser, DistCpConstants.INVALID_ARGUMENT);
+ doTestPreserveRawXAttrs(rawSrcName, rawDestName, options,
+ true, expectUser, DistCpConstants.SUCCESS);
+ }
+
private void doTestPreserveRawXAttrs(String src, String dest,
String preserveOpts, boolean expectRaw, boolean expectUser,
int expectedExitCode) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a85d079/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
index 2721638..624f7d5 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/DistCpTestUtils.java
@@ -18,21 +18,20 @@
package org.apache.hadoop.tools.util;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.util.ToolRunner;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
/**
* Utility class for DistCpTests
*/
@@ -79,10 +78,19 @@ public class DistCpTestUtils {
public static void assertRunDistCp(int exitCode, String src, String dst,
String options, Configuration conf)
throws Exception {
+ assertRunDistCp(exitCode, src, dst,
+ options == null ? new String[0] : options.trim().split(" "), conf);
+ }
+
+ private static void assertRunDistCp(int exitCode, String src, String dst,
+ String[] options, Configuration conf)
+ throws Exception {
DistCp distCp = new DistCp(conf, null);
- String[] optsArr = options == null ?
- new String[] { src, dst } :
- new String[] { options, src, dst };
+ String[] optsArr = new String[options.length + 2];
+ System.arraycopy(options, 0, optsArr, 0, options.length);
+ optsArr[optsArr.length - 2] = src;
+ optsArr[optsArr.length - 1] = dst;
+
assertEquals(exitCode,
ToolRunner.run(conf, distCp, optsArr));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org