You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2019/10/03 23:23:57 UTC
[hadoop] branch branch-3.1 updated (ddea9d6 -> e68ccca)
This is an automated email from the ASF dual-hosted git repository.
shv pushed a change to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from ddea9d6 HDFS-14064. WEBHDFS: Support Enable/Disable EC Policy. Contributed by Ayush Saxena.
new 8ee3a87 HDFS-14494. Move Server logging of StatedId inside receiveRequestState(). Contributed by Shweta Yakkali.
new 7ca418c HDFS-14660. [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider. Contributed by Chao Sun.
new e68ccca HDFS-14262. [SBN read] Make Log.WARN message in GlobalStateIdContext more informative. Contributed by Shweta Yakkali.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hdfs/server/namenode/GlobalStateIdContext.java | 28 +++++++++++++---
.../namenode/ha/TestConsistentReadsObserver.java | 37 ++++++++++++++++++++++
2 files changed, 60 insertions(+), 5 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-14262. [SBN read] Make Log.WARN message in
GlobalStateIdContext more informative. Contributed by Shweta Yakkali.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e68ccca2d7b16a3238f81b24b0f7108371fbac8e
Author: Shweta Yakkali <sh...@cloudera.com>
AuthorDate: Thu Feb 14 00:22:13 2019 -0800
HDFS-14262. [SBN read] Make Log.WARN message in GlobalStateIdContext more informative. Contributed by Shweta Yakkali.
---
.../apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
index ca1c813..7d61359 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
@@ -145,8 +145,10 @@ class GlobalStateIdContext implements AlignmentContext {
if (clientStateId > serverStateId &&
HAServiceState.ACTIVE.equals(namesystem.getState())) {
- FSNamesystem.LOG.warn("A client sent stateId: " + clientStateId +
- ", but server state is: " + serverStateId);
+ FSNamesystem.LOG.warn("The client stateId: {} is greater than "
+ + "the server stateId: {} This is unexpected. "
+ + "Resetting client stateId to server stateId",
+ clientStateId, serverStateId);
return serverStateId;
}
if (HAServiceState.OBSERVER.equals(namesystem.getState()) &&
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDFS-14494. Move Server logging of StatedId inside
receiveRequestState(). Contributed by Shweta Yakkali.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8ee3a8768d858c816e18e411812707f4581a9be7
Author: Shweta Yakkali <sh...@cloudera.com>
AuthorDate: Mon Jun 10 10:22:16 2019 -0700
HDFS-14494. Move Server logging of StatedId inside receiveRequestState(). Contributed by Shweta Yakkali.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
---
.../apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
index 0da8995..9320748 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
@@ -124,9 +124,11 @@ class GlobalStateIdContext implements AlignmentContext {
@Override
public long receiveRequestState(RpcRequestHeaderProto header,
long clientWaitTime) throws RetriableException {
- long serverStateId =
- namesystem.getFSImage().getCorrectLastAppliedOrWrittenTxId();
+ long serverStateId = getLastSeenStateId();
long clientStateId = header.getStateId();
+ FSNamesystem.LOG.trace("Client State ID= {} and Server State ID= {}",
+ clientStateId, serverStateId);
+
if (clientStateId > serverStateId &&
HAServiceState.ACTIVE.equals(namesystem.getState())) {
FSNamesystem.LOG.warn("A client sent stateId: " + clientStateId +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDFS-14660. [SBN Read] ObserverNameNode should
throw StandbyException for requests not from ObserverProxyProvider.
Contributed by Chao Sun.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7ca418c2c00ee50f308655d44b5af12d42ca62f0
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Sun Jul 28 08:41:42 2019 +0530
HDFS-14660. [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider. Contributed by Chao Sun.
---
.../hdfs/server/namenode/GlobalStateIdContext.java | 16 +++++++++-
.../namenode/ha/TestConsistentReadsObserver.java | 37 ++++++++++++++++++++++
2 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
index 9320748..ca1c813 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GlobalStateIdContext.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.concurrent.TimeUnit;
@@ -26,9 +27,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly;
import org.apache.hadoop.ipc.AlignmentContext;
import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
@@ -123,7 +126,18 @@ class GlobalStateIdContext implements AlignmentContext {
*/
@Override
public long receiveRequestState(RpcRequestHeaderProto header,
- long clientWaitTime) throws RetriableException {
+ long clientWaitTime) throws IOException {
+ if (!header.hasStateId() &&
+ HAServiceState.OBSERVER.equals(namesystem.getState())) {
+ // This could happen if client configured with non-observer proxy provider
+ // (e.g., ConfiguredFailoverProxyProvider) is accessing a cluster with
+ // observers. In this case, we should let the client failover to the
+ // active node, rather than potentially serving stale result (client
+ // stateId is 0 if not set).
+ throw new StandbyException("Observer Node received request without "
+ + "stateId. This mostly likely is because client is not configured "
+ + "with " + ObserverReadProxyProvider.class.getSimpleName());
+ }
long serverStateId = getLastSeenStateId();
long clientStateId = header.getStateId();
FSNamesystem.LOG.trace("Client State ID= {} and Server State ID= {}",
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
index 2f8f115..9606785 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.Collections;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,12 +35,15 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RpcScheduler;
import org.apache.hadoop.ipc.Schedulable;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.After;
@@ -346,6 +350,39 @@ public class TestConsistentReadsObserver {
reader.interrupt();
}
+ @Test
+ public void testRequestFromNonObserverProxyProvider() throws Exception {
+ // Create another HDFS client using ConfiguredFailoverProvider
+ Configuration conf2 = new Configuration(conf);
+
+ // Populate the above configuration with only a single observer in the
+ // namenode list. Also reduce retries to make test finish faster.
+ HATestUtil.setFailoverConfigurations(
+ conf2,
+ HATestUtil.getLogicalHostname(dfsCluster),
+ Collections.singletonList(
+ dfsCluster.getNameNode(2).getNameNodeAddress()),
+ ConfiguredFailoverProxyProvider.class);
+ conf2.setBoolean("fs.hdfs.impl.disable.cache", true);
+ conf2.setInt(HdfsClientConfigKeys.Retry.MAX_ATTEMPTS_KEY, 1);
+ conf2.setInt(HdfsClientConfigKeys.Failover.MAX_ATTEMPTS_KEY, 1);
+ FileSystem dfs2 = FileSystem.get(conf2);
+
+ dfs.mkdir(testPath, FsPermission.getDefault());
+ dfsCluster.rollEditLogAndTail(0);
+
+ try {
+ // Request should be rejected by observer and throw StandbyException
+ dfs2.listStatus(testPath);
+ fail("listStatus should have thrown exception");
+ } catch (RemoteException re) {
+ IOException e = re.unwrapRemoteException();
+ assertTrue("should have thrown StandbyException but got "
+ + e.getClass().getSimpleName(),
+ e instanceof StandbyException);
+ }
+ }
+
private void assertSentTo(int nnIdx) throws IOException {
assertTrue("Request was not sent to the expected namenode " + nnIdx,
HATestUtil.isSentToAnyOfNameNodes(dfs, dfsCluster, nnIdx));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org