You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/11/27 20:41:01 UTC
[1/2] hadoop git commit: HADOOP-15932. Oozie unable to create
sharelib in s3a filesystem.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 838190482 -> fa1d4ba7d
refs/heads/trunk 300f772c3 -> 4c106fca0
HADOOP-15932. Oozie unable to create sharelib in s3a filesystem.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c106fca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c106fca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c106fca
Branch: refs/heads/trunk
Commit: 4c106fca0ca91536e288f11052568406a0b84300
Parents: 300f772
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 27 20:39:54 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 27 20:39:54 2018 +0000
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 8 +++++++-
.../hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java | 15 +++++++++++----
2 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c106fca/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 3c432fc..df7bd77 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -2357,7 +2357,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities {
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
- innerCopyFromLocalFile(delSrc, overwrite, src, dst);
+ entryPoint(INVOCATION_COPY_FROM_LOCAL_FILE);
+ LOG.debug("Copying local file from {} to {}", src, dst);
+// innerCopyFromLocalFile(delSrc, overwrite, src, dst);
+ super.copyFromLocalFile(delSrc, overwrite, src, dst);
}
/**
@@ -2367,6 +2370,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities {
* This version doesn't need to create a temporary file to calculate the md5.
* Sadly this doesn't seem to be used by the shell cp :(
*
+ * <i>HADOOP-15932:</i> this method has been unwired from
+ * {@link #copyFromLocalFile(boolean, boolean, Path, Path)} until
+ * it is extended to list and copy whole directories.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c106fca/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
index 7dc286d..668e129 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@ -22,23 +22,27 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.junit.Ignore;
import org.junit.Test;
-import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathExistsException;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
+ * Some of the tests have been disabled pending a fix for HADOOP-15932 and
+ * recursive directory copying; the test cases themselves may be obsolete.
*/
public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
- private static final Charset ASCII = Charsets.US_ASCII;
+ private static final Charset ASCII = StandardCharsets.US_ASCII;
private File file;
@@ -80,7 +84,8 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
public void testCopyFileNoOverwrite() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
- intercept(FileAlreadyExistsException.class,
+ // HADOOP-15932: the exception type changes here
+ intercept(PathExistsException.class,
() -> upload(file, false));
}
@@ -95,6 +100,7 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
}
@Test
+ @Ignore("HADOOP-15932")
public void testCopyFileNoOverwriteDirectory() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
@@ -110,11 +116,12 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
file = File.createTempFile("test", ".txt");
file.delete();
// first upload to create
- intercept(FileNotFoundException.class, "No file",
+ intercept(FileNotFoundException.class, "",
() -> upload(file, true));
}
@Test
+ @Ignore("HADOOP-15932")
public void testCopyDirectoryFile() throws Throwable {
file = File.createTempFile("test", ".txt");
// first upload to create
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-15932. Oozie unable to create
sharelib in s3a filesystem.
Posted by st...@apache.org.
HADOOP-15932. Oozie unable to create sharelib in s3a filesystem.
Contributed by Steve Loughran.
(cherry picked from commit 4c106fca0ca91536e288f11052568406a0b84300)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa1d4ba7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa1d4ba7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa1d4ba7
Branch: refs/heads/branch-3.2
Commit: fa1d4ba7d441ffc25e5554e7fbd86e4b1c561a59
Parents: 8381904
Author: Steve Loughran <st...@apache.org>
Authored: Tue Nov 27 20:40:48 2018 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Nov 27 20:40:48 2018 +0000
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 8 +++++++-
.../hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java | 15 +++++++++++----
2 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa1d4ba7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index e817f0d..67d83c0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -2350,7 +2350,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities {
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
- innerCopyFromLocalFile(delSrc, overwrite, src, dst);
+ entryPoint(INVOCATION_COPY_FROM_LOCAL_FILE);
+ LOG.debug("Copying local file from {} to {}", src, dst);
+// innerCopyFromLocalFile(delSrc, overwrite, src, dst);
+ super.copyFromLocalFile(delSrc, overwrite, src, dst);
}
/**
@@ -2360,6 +2363,9 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities {
* This version doesn't need to create a temporary file to calculate the md5.
* Sadly this doesn't seem to be used by the shell cp :(
*
+ * <i>HADOOP-15932:</i> this method has been unwired from
+ * {@link #copyFromLocalFile(boolean, boolean, Path, Path)} until
+ * it is extended to list and copy whole directories.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa1d4ba7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
index 7dc286d..668e129 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@ -22,23 +22,27 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.junit.Ignore;
import org.junit.Test;
-import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathExistsException;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
+ * Some of the tests have been disabled pending a fix for HADOOP-15932 and
+ * recursive directory copying; the test cases themselves may be obsolete.
*/
public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
- private static final Charset ASCII = Charsets.US_ASCII;
+ private static final Charset ASCII = StandardCharsets.US_ASCII;
private File file;
@@ -80,7 +84,8 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
public void testCopyFileNoOverwrite() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
- intercept(FileAlreadyExistsException.class,
+ // HADOOP-15932: the exception type changes here
+ intercept(PathExistsException.class,
() -> upload(file, false));
}
@@ -95,6 +100,7 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
}
@Test
+ @Ignore("HADOOP-15932")
public void testCopyFileNoOverwriteDirectory() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
@@ -110,11 +116,12 @@ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
file = File.createTempFile("test", ".txt");
file.delete();
// first upload to create
- intercept(FileNotFoundException.class, "No file",
+ intercept(FileNotFoundException.class, "",
() -> upload(file, true));
}
@Test
+ @Ignore("HADOOP-15932")
public void testCopyDirectoryFile() throws Throwable {
file = File.createTempFile("test", ".txt");
// first upload to create
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org