You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2020/10/13 01:05:03 UTC
[hadoop] branch trunk updated: HDFS-15567. [SBN Read] HDFS should
expose msync() API to allow downstream applications call it explicitly.
Contributed by Konstantin V Shvachko.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b3786d6 HDFS-15567. [SBN Read] HDFS should expose msync() API to allow downstream applications call it explicitly. Contributed by Konstantin V Shvachko.
b3786d6 is described below
commit b3786d6c3cc13b0b92b9f42da1731c4ce35c9ded
Author: Konstantin V Shvachko <sh...@apache.org>
AuthorDate: Mon Oct 12 17:26:24 2020 -0700
HDFS-15567. [SBN Read] HDFS should expose msync() API to allow downstream applications call it explicitly. Contributed by Konstantin V Shvachko.
---
.../org/apache/hadoop/fs/AbstractFileSystem.java | 13 +++++++
.../java/org/apache/hadoop/fs/FileContext.java | 10 ++++++
.../main/java/org/apache/hadoop/fs/FileSystem.java | 13 +++++++
.../org/apache/hadoop/fs/FilterFileSystem.java | 5 +++
.../main/java/org/apache/hadoop/fs/FilterFs.java | 5 +++
.../java/org/apache/hadoop/fs/HarFileSystem.java | 5 +++
.../src/main/java/org/apache/hadoop/fs/Hdfs.java | 13 ++++++-
.../apache/hadoop/hdfs/DistributedFileSystem.java | 11 ++++++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +-
.../namenode/ha/TestConsistentReadsObserver.java | 40 ++++++++++++++++++++--
10 files changed, 114 insertions(+), 4 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index ec346b4..e38405b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -865,6 +865,19 @@ public abstract class AbstractFileSystem implements PathCapabilities {
UnresolvedLinkException, IOException;
/**
+ * Synchronize client metadata state.
+ * <p/>In some FileSystem implementations such as HDFS metadata
+ * synchronization is essential to guarantee consistency of read requests
+ * particularly in HA setting.
+ * @throws IOException
+ * @throws UnsupportedOperationException
+ */
+ public void msync() throws IOException, UnsupportedOperationException {
+ throw new UnsupportedOperationException(getClass().getCanonicalName() +
+ " does not support method msync");
+ }
+
+ /**
* The specification of this method matches that of
* {@link FileContext#access(Path, FsAction)}
* except that an UnresolvedLinkException may be thrown if a symlink is
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index e5f4ef3..c3aa9c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -1250,6 +1250,16 @@ public class FileContext implements PathCapabilities {
}
/**
+ * Synchronize client metadata state.
+ *
+ * @throws IOException
+ * @throws UnsupportedOperationException
+ */
+ public void msync() throws IOException, UnsupportedOperationException {
+ defaultFS.msync();
+ }
+
+ /**
* Checks if the user can access a path. The mode specifies which access
* checks to perform. If the requested permissions are granted, then the
* method returns normally. If access is denied, then the method throws an
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 9927247..eee871e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2676,6 +2676,19 @@ public abstract class FileSystem extends Configured
public abstract FileStatus getFileStatus(Path f) throws IOException;
/**
+ * Synchronize client metadata state.
+ * <p/>In some FileSystem implementations such as HDFS metadata
+ * synchronization is essential to guarantee consistency of read requests
+ * particularly in HA setting.
+ * @throws IOException
+ * @throws UnsupportedOperationException
+ */
+ public void msync() throws IOException, UnsupportedOperationException {
+ throw new UnsupportedOperationException(getClass().getCanonicalName() +
+ " does not support method msync");
+ }
+
+ /**
* Checks if the user can access a path. The mode specifies which access
* checks to perform. If the requested permissions are granted, then the
* method returns normally. If access is denied, then the method throws an
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 4241097..607aa26 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -463,6 +463,11 @@ public class FilterFileSystem extends FileSystem {
}
@Override
+ public void msync() throws IOException, UnsupportedOperationException {
+ fs.msync();
+ }
+
+ @Override
public void access(Path path, FsAction mode) throws AccessControlException,
FileNotFoundException, IOException {
fs.access(path, mode);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 27e75d8..7d979b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -125,6 +125,11 @@ public abstract class FilterFs extends AbstractFileSystem {
}
@Override
+ public void msync() throws IOException, UnsupportedOperationException {
+ myFs.msync();
+ }
+
+ @Override
public void access(Path path, FsAction mode) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
checkPath(path);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index 5f4c4a2..02c0916 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -676,6 +676,11 @@ public class HarFileSystem extends FileSystem {
return hstatus;
}
+ @Override
+ public void msync() throws IOException, UnsupportedOperationException {
+ fs.msync();
+ }
+
/**
* @return null since no checksum algorithm is implemented.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 4162b19..60ce8cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -153,7 +153,18 @@ public class Hdfs extends AbstractFileSystem {
throw new FileNotFoundException("File does not exist: " + f.toString());
}
}
-
+
+ /**
+ * Synchronize client metadata state with Active NameNode.
+ * <p/>In HA the client synchronizes its state with the Active NameNode
+ * in order to guarantee subsequent read consistency from Observer Nodes.
+ * @throws IOException
+ */
+ @Override
+ public void msync() throws IOException {
+ dfs.msync();
+ }
+
@Override
public FileStatus getFileLinkStatus(Path f)
throws IOException, UnresolvedLinkException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 9b08434..1a6422a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1767,6 +1767,17 @@ public class DistributedFileSystem extends FileSystem
}.resolve(this, absF);
}
+ /**
+ * Synchronize client metadata state with Active NameNode.
+ * <p/>In HA the client synchronizes its state with the Active NameNode
+ * in order to guarantee subsequent read consistency from Observer Nodes.
+ * @throws IOException
+ */
+ @Override
+ public void msync() throws IOException {
+ dfs.msync();
+ }
+
@SuppressWarnings("deprecation")
@Override
public void createSymlink(final Path target, final Path link,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 7c9c899..a2a9066 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2679,7 +2679,8 @@ public class MiniDFSCluster implements AutoCloseable {
public void rollEditLogAndTail(int nnIndex) throws Exception {
getNameNode(nnIndex).getRpcServer().rollEditLog();
for (int i = 2; i < getNumNameNodes(); i++) {
- getNameNode(i).getNamesystem().getEditLogTailer().doTailEdits();
+ long el = getNameNode(i).getNamesystem().getEditLogTailer().doTailEdits();
+ LOG.info("editsLoaded {}", el);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
index 9606785..854027a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
@@ -30,9 +31,12 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -108,7 +112,8 @@ public class TestConsistentReadsObserver {
final int observerIdx = 2;
NameNode nn = dfsCluster.getNameNode(observerIdx);
int port = nn.getNameNodeAddress().getPort();
- Configuration configuration = dfsCluster.getConfiguration(observerIdx);
+ Configuration originalConf = dfsCluster.getConfiguration(observerIdx);
+ Configuration configuration = new Configuration(originalConf);
String prefix = CommonConfigurationKeys.IPC_NAMESPACE + "." + port + ".";
configuration.set(prefix + CommonConfigurationKeys.IPC_SCHEDULER_IMPL_KEY,
TestRpcScheduler.class.getName());
@@ -125,6 +130,8 @@ public class TestConsistentReadsObserver {
// be triggered and client should retry active NN.
dfs.getFileStatus(testPath);
assertSentTo(0);
+ // reset the original call queue
+ NameNodeAdapter.getRpcServer(nn).refreshCallQueue(originalConf);
}
@Test
@@ -194,7 +201,7 @@ public class TestConsistentReadsObserver {
// Therefore, the subsequent getFileStatus call should succeed.
if (!autoMsync) {
// If not testing auto-msync, perform an explicit one here
- dfs2.getClient().msync();
+ dfs2.msync();
} else if (autoMsyncPeriodMs > 0) {
Thread.sleep(autoMsyncPeriodMs);
}
@@ -383,6 +390,35 @@ public class TestConsistentReadsObserver {
}
}
+ @Test(timeout=10000)
+ public void testMsyncFileContext() throws Exception {
+ NameNode nn0 = dfsCluster.getNameNode(0);
+ NameNode nn2 = dfsCluster.getNameNode(2);
+ HAServiceStatus st = nn0.getRpcServer().getServiceStatus();
+ assertEquals("nn0 is not active", HAServiceState.ACTIVE, st.getState());
+ st = nn2.getRpcServer().getServiceStatus();
+ assertEquals("nn2 is not observer", HAServiceState.OBSERVER, st.getState());
+
+ FileContext fc = FileContext.getFileContext(conf);
+ // initialize observer proxy for FileContext
+ fc.getFsStatus(testPath);
+
+ Path p = new Path(testPath, "testMsyncFileContext");
+ fc.mkdir(p, FsPermission.getDefault(), true);
+ fc.msync();
+ dfsCluster.rollEditLogAndTail(0);
+ LOG.info("State id active = {}, Stat id observer = {}",
+ nn0.getNamesystem().getFSImage().getLastAppliedOrWrittenTxId(),
+ nn2.getNamesystem().getFSImage().getLastAppliedOrWrittenTxId());
+ try {
+ // if getFileStatus is taking too long due to server requeueing
+ // the test will time out
+ fc.getFileStatus(p);
+ } catch (FileNotFoundException e) {
+ fail("File should exist on Observer after msync");
+ }
+ }
+
private void assertSentTo(int nnIdx) throws IOException {
assertTrue("Request was not sent to the expected namenode " + nnIdx,
HATestUtil.isSentToAnyOfNameNodes(dfs, dfsCluster, nnIdx));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org