You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2022/01/05 18:23:22 UTC
[hadoop] branch trunk updated: HADOOP-18056. DistCp: Filter duplicates in the source paths. (#3825). Contributed by Ayush Saxena.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 657a288 HADOOP-18056. DistCp: Filter duplicates in the source paths. (#3825). Contributed by Ayush Saxena.
657a288 is described below
commit 657a2882e93fadb4cd895bc3fdf1ae616514c03d
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Wed Jan 5 23:53:07 2022 +0530
HADOOP-18056. DistCp: Filter duplicates in the source paths. (#3825). Contributed by Ayush Saxena.
Reviewed-by: tomscut <li...@bigo.sg>
Reviewed-by: Steve Loughran <st...@apache.org>
---
.../org/apache/hadoop/tools/DistCpOptions.java | 15 +++++++++++++-
.../org/apache/hadoop/tools/TestCopyListing.java | 23 ++++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index c9f2ab2..fee53ec 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -30,8 +30,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.util.DistCpUtils;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
@@ -234,7 +236,18 @@ public final class DistCpOptions {
public List<Path> getSourcePaths() {
return sourcePaths == null ?
- null : Collections.unmodifiableList(sourcePaths);
+ null :
+ Collections.unmodifiableList(getUniquePaths(sourcePaths));
+ }
+
+ private List<Path> getUniquePaths(List<Path> srcPaths) {
+ Set<Path> uniquePaths = new LinkedHashSet<>();
+ for (Path path : srcPaths) {
+ if (!uniquePaths.add(path)) {
+ LOG.info("Path: {} added multiple times, ignoring the redundant entry.", path);
+ }
+ }
+ return new ArrayList<>(uniquePaths);
}
public Path getTargetPath() {
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
index ce7d00d..69e1421 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
@@ -167,6 +167,29 @@ public class TestCopyListing extends SimpleCopyListing {
}
}
+ @Test
+ public void testDuplicateSourcePaths() throws Exception {
+ FileSystem fs = FileSystem.get(getConf());
+ List<Path> srcPaths = new ArrayList<Path>();
+ try {
+ srcPaths.add(new Path("/tmp/in"));
+ srcPaths.add(new Path("/tmp/in"));
+ TestDistCpUtils.createFile(fs, "/tmp/in/src1/1.txt");
+ TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt");
+ Path target = new Path("/tmp/out");
+ Path listingFile = new Path("/tmp/list");
+ final DistCpOptions options =
+ new DistCpOptions.Builder(srcPaths, target).build();
+ final DistCpContext context = new DistCpContext(options);
+ CopyListing listing =
+ CopyListing.getCopyListing(getConf(), CREDENTIALS, context);
+ listing.buildListing(listingFile, context);
+ Assert.assertTrue(fs.exists(listingFile));
+ } finally {
+ TestDistCpUtils.delete(fs, "/tmp");
+ }
+ }
+
@Test(timeout=10000)
public void testBuildListing() {
FileSystem fs = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org