You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2019/06/28 23:06:04 UTC
[hadoop] 01/50: HDFS-12975. [SBN read] Changes to the NameNode to
support reads from standby. Contributed by Chao Sun.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3932ac4ef796ad7ee6489dbe20dc7f9c2939aa86
Author: Chao Sun <su...@apache.org>
AuthorDate: Tue Mar 20 18:37:59 2018 -0700
HDFS-12975. [SBN read] Changes to the NameNode to support reads from standby. Contributed by Chao Sun.
---
.../hdfs/server/common/HdfsServerConstants.java | 4 +++-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 14 ++++++-----
.../hadoop/hdfs/server/namenode/NameNode.java | 28 ++++++++++++++++++++--
.../hdfs/server/namenode/NameNodeRpcServer.java | 2 +-
.../hdfs/server/namenode/ha/StandbyState.java | 17 +++++++++++++
5 files changed, 55 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index c6e2263..21ed9f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -157,7 +157,9 @@ public interface HdfsServerConstants {
// only used for StorageDirectory.analyzeStorage() in hot swap drive scenario.
// TODO refactor StorageDirectory.analyzeStorage() so that we can do away with
// this in StartupOption.
- HOTSWAP("-hotswap");
+ HOTSWAP("-hotswap"),
+ // Startup the namenode in observer mode.
+ OBSERVER("-observer");
private static final Pattern ENUM_WITH_ROLLING_UPGRADE_OPTION = Pattern.compile(
"(\\w+)\\((\\w+)\\)");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d1b882c..42259c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -539,7 +539,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private final ReentrantLock cpLock;
/**
- * Used when this NN is in standby state to read from the shared edit log.
+ * Used when this NN is in standby or observer state to read from the
+ * shared edit log.
*/
private EditLogTailer editLogTailer = null;
@@ -1376,24 +1377,25 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
- * Start services required in standby state
+ * Start services required in standby or observer state
*
* @throws IOException
*/
- void startStandbyServices(final Configuration conf) throws IOException {
- LOG.info("Starting services required for standby state");
+ void startStandbyServices(final Configuration conf, boolean isObserver)
+ throws IOException {
+ LOG.info("Starting services required for " +
+ (isObserver ? "observer" : "standby") + " state");
if (!getFSImage().editLog.isOpenForRead()) {
// During startup, we're already open for read.
getFSImage().editLog.initSharedJournalsForRead();
}
-
blockManager.setPostponeBlocksFromFuture(true);
// Disable quota checks while in standby.
dir.disableQuotaChecks();
editLogTailer = new EditLogTailer(this, conf);
editLogTailer.start();
- if (standbyShouldCheckpoint) {
+ if (!isObserver && standbyShouldCheckpoint) {
standbyCheckpointer = new StandbyCheckpointer(conf, this);
standbyCheckpointer.start();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 38e9e7e..f98ae80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -373,6 +373,7 @@ public class NameNode extends ReconfigurableBase implements
LoggerFactory.getLogger("BlockStateChange");
public static final HAState ACTIVE_STATE = new ActiveState();
public static final HAState STANDBY_STATE = new StandbyState();
+ public static final HAState OBSERVER_STATE = new StandbyState(true);
private static final String NAMENODE_HTRACE_PREFIX = "namenode.htrace.";
@@ -976,9 +977,11 @@ public class NameNode extends ReconfigurableBase implements
}
protected HAState createHAState(StartupOption startOpt) {
- if (!haEnabled || startOpt == StartupOption.UPGRADE
+ if (!haEnabled || startOpt == StartupOption.UPGRADE
|| startOpt == StartupOption.UPGRADEONLY) {
return ACTIVE_STATE;
+ } else if (startOpt == StartupOption.OBSERVER) {
+ return OBSERVER_STATE;
} else {
return STANDBY_STATE;
}
@@ -1444,6 +1447,8 @@ public class NameNode extends ReconfigurableBase implements
startOpt = StartupOption.BACKUP;
} else if (StartupOption.CHECKPOINT.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.CHECKPOINT;
+ } else if (StartupOption.OBSERVER.getName().equalsIgnoreCase(cmd)) {
+ startOpt = StartupOption.OBSERVER;
} else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)
|| StartupOption.UPGRADEONLY.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd) ?
@@ -1757,6 +1762,11 @@ public class NameNode extends ReconfigurableBase implements
if (!haEnabled) {
throw new ServiceFailedException("HA for namenode is not enabled");
}
+ if (state == OBSERVER_STATE) {
+ // TODO: we may need to remove this when enabling failover for observer
+ throw new ServiceFailedException(
+ "Cannot transition from Observer to Active");
+ }
state.setState(haContext, ACTIVE_STATE);
}
@@ -1766,6 +1776,11 @@ public class NameNode extends ReconfigurableBase implements
if (!haEnabled) {
throw new ServiceFailedException("HA for namenode is not enabled");
}
+ if (state == OBSERVER_STATE) {
+ // TODO: we may need to remove this when enabling failover for observer
+ throw new ServiceFailedException(
+ "Cannot transition from Observer to Standby");
+ }
state.setState(haContext, STANDBY_STATE);
}
@@ -1820,6 +1835,7 @@ public class NameNode extends ReconfigurableBase implements
@Override // NameNodeStatusMXBean
public String getState() {
+ // TODO: maybe we should return a different result for observer namenode?
String servStateStr = "";
HAServiceState servState = getServiceState();
if (null != servState) {
@@ -1920,7 +1936,8 @@ public class NameNode extends ReconfigurableBase implements
@Override
public void startStandbyServices() throws IOException {
try {
- namesystem.startStandbyServices(getConf());
+ namesystem.startStandbyServices(getConf(),
+ state == NameNode.OBSERVER_STATE);
} catch (Throwable t) {
doImmediateShutdown(t);
}
@@ -1967,6 +1984,9 @@ public class NameNode extends ReconfigurableBase implements
@Override
public boolean allowStaleReads() {
+ if (state == OBSERVER_STATE) {
+ return true;
+ }
return allowStaleStandbyReads;
}
@@ -1980,6 +2000,10 @@ public class NameNode extends ReconfigurableBase implements
return (state.equals(ACTIVE_STATE));
}
+ public boolean isObserverState() {
+ return state.equals(OBSERVER_STATE);
+ }
+
/**
* Returns whether the NameNode is completely started
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6d21742..3c7d8f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1565,7 +1565,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
if (nn.getFSImage().isUpgradeFinalized() &&
!namesystem.isRollingUpgrade() &&
- !nn.isStandbyState() &&
+ nn.isActiveState() &&
noStaleStorages) {
return new FinalizeCommand(poolId);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyState.java
index d782bdf..9a21888 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyState.java
@@ -39,8 +39,15 @@ import org.apache.hadoop.ipc.StandbyException;
*/
@InterfaceAudience.Private
public class StandbyState extends HAState {
+ private final boolean isObserver;
+
public StandbyState() {
+ this(false);
+ }
+
+ public StandbyState(boolean isObserver) {
super(HAServiceState.STANDBY);
+ this.isObserver = isObserver;
}
@Override
@@ -49,6 +56,11 @@ public class StandbyState extends HAState {
setStateInternal(context, s);
return;
}
+ if (isObserver && s == NameNode.STANDBY_STATE) {
+ // To guard against the exception in the following super call.
+ // The other case, standby -> observer, should not happen.
+ return;
+ }
super.setState(context, s);
}
@@ -92,5 +104,10 @@ public class StandbyState extends HAState {
public boolean shouldPopulateReplQueues() {
return false;
}
+
+ @Override
+ public String toString() {
+ return isObserver ? "observer" : "standby";
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org