You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ay...@apache.org on 2020/09/18 10:09:12 UTC
[hadoop] branch branch-3.3 updated (2d9c539 -> 2ce5846)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 2d9c539 HDFS-15578: Fix the rename issues with fallback fs enabled (#2305). Contributed by Uma Maheswara Rao G.
new 1fc1b34 HDFS-15558: ViewDistributedFileSystem#recoverLease should call super.recoverLease when there are no mounts configured (#2275) Contributed by Uma Maheswara Rao G.
new 2ce5846 HDFS-15585: ViewDFS#getDelegationToken should not throw UnsupportedOperationException. (#2312). Contributed by Uma Maheswara Rao G.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hdfs/ViewDistributedFileSystem.java | 22 +++++++++--
.../org/apache/hadoop/hdfs/TestLeaseRecovery.java | 16 +++++++-
.../hadoop/hdfs/TestViewDistributedFileSystem.java | 45 ++++++++++++++++++++++
3 files changed, 79 insertions(+), 4 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDFS-15558: ViewDistributedFileSystem#recoverLease
should call super.recoverLease when there are no mounts configured (#2275)
Contributed by Uma Maheswara Rao G.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1fc1b346336bea372b2d5d97756b121d732bfae6
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Mon Sep 7 11:36:13 2020 -0700
HDFS-15558: ViewDistributedFileSystem#recoverLease should call super.recoverLease when there are no mounts configured (#2275) Contributed by Uma Maheswara Rao G.
---
.../hadoop/hdfs/ViewDistributedFileSystem.java | 7 +++++++
.../org/apache/hadoop/hdfs/TestLeaseRecovery.java | 16 ++++++++++++++-
.../hadoop/hdfs/TestViewDistributedFileSystem.java | 23 ++++++++++++++++++++++
3 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
index 0a68169..1afb5d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
@@ -266,6 +266,10 @@ public class ViewDistributedFileSystem extends DistributedFileSystem {
@Override
public boolean recoverLease(final Path f) throws IOException {
+ if (this.vfs == null) {
+ return super.recoverLease(f);
+ }
+
ViewFileSystemOverloadScheme.MountPathInfo<FileSystem> mountPathInfo =
this.vfs.getMountPathInfo(f, getConf());
checkDFS(mountPathInfo.getTargetFs(), "recoverLease");
@@ -286,6 +290,9 @@ public class ViewDistributedFileSystem extends DistributedFileSystem {
@Override
public FSDataInputStream open(PathHandle fd, int bufferSize)
throws IOException {
+ if (this.vfs == null) {
+ return super.open(fd, bufferSize);
+ }
return this.vfs.open(fd, bufferSize);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
index a1cce3e..399aa1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
@@ -280,8 +280,22 @@ public class TestLeaseRecovery {
*/
@Test
public void testLeaseRecoveryAndAppend() throws Exception {
+ testLeaseRecoveryAndAppend(new Configuration());
+ }
+
+ /**
+ * Recover the lease on a file and append file from another client with
+ * ViewDFS enabled.
+ */
+ @Test
+ public void testLeaseRecoveryAndAppendWithViewDFS() throws Exception {
Configuration conf = new Configuration();
- try{
+ conf.set("fs.hdfs.impl", ViewDistributedFileSystem.class.getName());
+ testLeaseRecoveryAndAppend(conf);
+ }
+
+ private void testLeaseRecoveryAndAppend(Configuration conf) throws Exception {
+ try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
Path file = new Path("/testLeaseRecovery");
DistributedFileSystem dfs = cluster.getFileSystem();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
index 3c5a0be..0ba0841 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
@@ -17,9 +17,13 @@
*/
package org.apache.hadoop.hdfs;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathHandle;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.test.Whitebox;
+import org.junit.Test;
import java.io.IOException;
@@ -44,4 +48,23 @@ public class TestViewDistributedFileSystem extends TestDistributedFileSystem{
data.set(null);
super.testStatistics();
}
+
+ @Test
+ public void testOpenWithPathHandle() throws Exception {
+ Configuration conf = getTestConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ FileSystem fileSys = cluster.getFileSystem();
+ Path openTestPath = new Path("/testOpen");
+ fileSys.create(openTestPath).close();
+ PathHandle pathHandle =
+ fileSys.getPathHandle(fileSys.getFileStatus(openTestPath));
+ fileSys.open(pathHandle, 1024).close();
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-15585: ViewDFS#getDelegationToken should not
throw UnsupportedOperationException. (#2312). Contributed by Uma Maheswara
Rao G.
Posted by ay...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 2ce5846bfa58b6bb3b4c15313c1a6cc9baaaed8f
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Fri Sep 18 02:48:10 2020 -0700
HDFS-15585: ViewDFS#getDelegationToken should not throw UnsupportedOperationException. (#2312). Contributed by Uma Maheswara Rao G.
---
.../hadoop/hdfs/ViewDistributedFileSystem.java | 15 ++++++++++++---
.../hadoop/hdfs/TestViewDistributedFileSystem.java | 22 ++++++++++++++++++++++
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
index 1afb5d9..4fee963 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
@@ -1032,15 +1032,24 @@ public class ViewDistributedFileSystem extends DistributedFileSystem {
return super.getDefaultPort();
}
+ /**
+ * If no mount points configured, it works same as
+ * {@link DistributedFileSystem#getDelegationToken(String)}. If
+ * there are mount points configured and if default fs(linkFallback)
+ * configured, then it will return default fs delegation token. Otherwise
+ * it will return null.
+ */
@Override
public Token<DelegationTokenIdentifier> getDelegationToken(String renewer)
throws IOException {
if (this.vfs == null) {
return super.getDelegationToken(renewer);
}
- //Let applications call getDelegationTokenIssuers and get respective
- // delegation tokens from child fs.
- throw new UnsupportedOperationException();
+
+ if (defaultDFS != null) {
+ return defaultDFS.getDelegationToken(renewer);
+ }
+ return null;
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
index 0ba0841..da0cb59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestViewDistributedFileSystem.java
@@ -18,14 +18,17 @@
package org.apache.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathHandle;
+import org.apache.hadoop.fs.viewfs.ConfigUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.test.Whitebox;
import org.junit.Test;
import java.io.IOException;
+import java.net.URI;
public class TestViewDistributedFileSystem extends TestDistributedFileSystem{
@Override
@@ -67,4 +70,23 @@ public class TestViewDistributedFileSystem extends TestDistributedFileSystem{
}
}
}
+
+ @Override
+ public void testEmptyDelegationToken() throws IOException {
+ Configuration conf = getTestConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ URI defaultUri =
+ URI.create(conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY));
+ ConfigUtil.addLinkFallback(conf, defaultUri.getHost(), defaultUri);
+ try (FileSystem fileSys = FileSystem.get(conf)) {
+ fileSys.getDelegationToken("");
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org