You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cl...@apache.org on 2019/08/30 22:55:54 UTC
[hadoop] branch branch-2 updated: HDFS-14726. Fix JN
incompatibility issue in branch-2 due to backport of HDFS-10519.
Contributed by Chen Liang.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new f05d87b HDFS-14726. Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519. Contributed by Chen Liang.
f05d87b is described below
commit f05d87b32e5cda2d4d157342c0ec570e12438441
Author: Chen Liang <cl...@apache.org>
AuthorDate: Fri Aug 30 15:55:35 2019 -0700
HDFS-14726. Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519. Contributed by Chen Liang.
---
.../org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 10 ++++++++--
.../hdfs/qjournal/client/QuorumJournalManager.java | 20 +++++++++++++++-----
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 2 +-
3 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index fa54330..478a4cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -292,8 +293,13 @@ public class PBHelper {
for (RemoteEditLogProto l : manifest.getLogsList()) {
logs.add(convert(l));
}
- return new RemoteEditLogManifest(logs,
- manifest.getCommittedTxnId());
+ long committedId = HdfsServerConstants.INVALID_TXID;
+ if (manifest.hasCommittedTxnId()) {
+ // An older version JN may not have this field, in which case committedId
+ // is set to INVALID_TXID.
+ committedId = manifest.getCommittedTxnId();
+ }
+ return new RemoteEditLogManifest(logs, committedId);
}
public static CheckpointCommandProto convert(CheckpointCommand cmd) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index b545fb2..14e63bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJourna
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
@@ -632,13 +633,22 @@ public class QuorumJournalManager implements JournalManager {
// If it's bounded by durable Txns, endTxId could not be larger
// than committedTxnId. This ensures the consistency.
+ // An older version JN may not return the field committedTxnId,
+ // in which case it is set to INVALID_TXID.
if (onlyDurableTxns && inProgressOk) {
- endTxId = Math.min(endTxId, committedTxnId);
- if (endTxId < remoteLog.getStartTxId()) {
- LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
- "the startTxId (" + remoteLog.getStartTxId() +
- ") - setting it to startTxId.");
+ if (committedTxnId == HdfsServerConstants.INVALID_TXID) {
+ LOG.warn("Received undefined committed txn id, "
+ + " NN and JN are on different version? "
+ + "- seting to startTxId");
endTxId = remoteLog.getStartTxId();
+ } else {
+ endTxId = Math.min(endTxId, committedTxnId);
+ if (endTxId < remoteLog.getStartTxId()) {
+ LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
+ "the startTxId (" + remoteLog.getStartTxId() +
+ ") - setting it to startTxId.");
+ endTxId = remoteLog.getStartTxId();
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
index 1825257..4c84615 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
@@ -92,7 +92,7 @@ message RemoteEditLogProto {
*/
message RemoteEditLogManifestProto {
repeated RemoteEditLogProto logs = 1;
- required uint64 committedTxnId = 2;
+ optional uint64 committedTxnId = 2;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org