You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2019/08/23 06:18:23 UTC
[hadoop] branch branch-3.1 updated: HDFS-13596. NN restart fails
after RollingUpgrade from 2.x to 3.x. Contributed by Fei Hui.
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b40aa71 HDFS-13596. NN restart fails after RollingUpgrade from 2.x to 3.x. Contributed by Fei Hui.
b40aa71 is described below
commit b40aa71cf04cbe7f4697132e30851c1e3a61db51
Author: Akira Ajisaka <aa...@apache.org>
AuthorDate: Fri Aug 23 14:44:34 2019 +0900
HDFS-13596. NN restart fails after RollingUpgrade from 2.x to 3.x. Contributed by Fei Hui.
(cherry picked from commit abc8fde4caea0e197568ee28392c46f1ce0d42e1)
---
.../hdfs/qjournal/client/QuorumJournalManager.java | 2 +-
.../hdfs/qjournal/client/QuorumOutputStream.java | 5 +-
.../hdfs/server/namenode/BackupJournalManager.java | 2 +-
.../server/namenode/EditLogBackupOutputStream.java | 8 ++-
.../server/namenode/EditLogFileOutputStream.java | 3 +-
.../hdfs/server/namenode/EditLogOutputStream.java | 16 +++++
.../hdfs/server/namenode/EditsDoubleBuffer.java | 10 +--
.../hadoop/hdfs/server/namenode/FSEditLog.java | 4 +-
.../hadoop/hdfs/server/namenode/FSEditLogOp.java | 22 ++++++-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 24 +++++++
.../apache/hadoop/hdfs/qjournal/QJMTestUtil.java | 4 +-
.../hadoop/hdfs/server/namenode/TestEditLog.java | 76 ++++++++++++++++++++++
.../server/namenode/TestEditsDoubleBuffer.java | 9 ++-
13 files changed, 163 insertions(+), 22 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 5f6d45f..7548e74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -433,7 +433,7 @@ public class QuorumJournalManager implements JournalManager {
loggers.waitForWriteQuorum(q, startSegmentTimeoutMs,
"startLogSegment(" + txId + ")");
return new QuorumOutputStream(loggers, txId, outputBufferCapacity,
- writeTxnsTimeoutMs);
+ writeTxnsTimeoutMs, layoutVersion);
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
index e094b21..26f978a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
@@ -36,17 +36,18 @@ class QuorumOutputStream extends EditLogOutputStream {
public QuorumOutputStream(AsyncLoggerSet loggers,
long txId, int outputBufferCapacity,
- int writeTimeoutMs) throws IOException {
+ int writeTimeoutMs, int logVersion) throws IOException {
super();
this.buf = new EditsDoubleBuffer(outputBufferCapacity);
this.loggers = loggers;
this.segmentTxId = txId;
this.writeTimeoutMs = writeTimeoutMs;
+ setCurrentLogVersion(logVersion);
}
@Override
public void write(FSEditLogOp op) throws IOException {
- buf.writeOp(op);
+ buf.writeOp(op, getCurrentLogVersion());
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
index e1ddfb9..880f079 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java
@@ -59,7 +59,7 @@ class BackupJournalManager implements JournalManager {
public EditLogOutputStream startLogSegment(long txId, int layoutVersion)
throws IOException {
EditLogBackupOutputStream stm = new EditLogBackupOutputStream(bnReg,
- journalInfo);
+ journalInfo, layoutVersion);
stm.startLogSegment(txId);
return stm;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
index 14d5b54..6f2ffa4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
@@ -53,8 +53,8 @@ class EditLogBackupOutputStream extends EditLogOutputStream {
private EditsDoubleBuffer doubleBuf;
EditLogBackupOutputStream(NamenodeRegistration bnReg, // backup node
- JournalInfo journalInfo) // active name-node
- throws IOException {
+ JournalInfo journalInfo, int logVersion) // active name-node
+ throws IOException {
super();
this.bnRegistration = bnReg;
this.journalInfo = journalInfo;
@@ -70,11 +70,12 @@ class EditLogBackupOutputStream extends EditLogOutputStream {
}
this.doubleBuf = new EditsDoubleBuffer(DEFAULT_BUFFER_SIZE);
this.out = new DataOutputBuffer(DEFAULT_BUFFER_SIZE);
+ setCurrentLogVersion(logVersion);
}
@Override // EditLogOutputStream
public void write(FSEditLogOp op) throws IOException {
- doubleBuf.writeOp(op);
+ doubleBuf.writeOp(op, getCurrentLogVersion());
}
@Override
@@ -89,6 +90,7 @@ class EditLogBackupOutputStream extends EditLogOutputStream {
public void create(int layoutVersion) throws IOException {
assert doubleBuf.isFlushed() : "previous data is not flushed yet";
this.doubleBuf = new EditsDoubleBuffer(DEFAULT_BUFFER_SIZE);
+ setCurrentLogVersion(layoutVersion);
}
@Override // EditLogOutputStream
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
index 830814c..200a6f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
@@ -94,7 +94,7 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
@Override
public void write(FSEditLogOp op) throws IOException {
- doubleBuf.writeOp(op);
+ doubleBuf.writeOp(op, getCurrentLogVersion());
}
/**
@@ -120,6 +120,7 @@ public class EditLogFileOutputStream extends EditLogOutputStream {
writeHeader(layoutVersion, doubleBuf.getCurrentBuf());
setReadyToFlush();
flush();
+ setCurrentLogVersion(layoutVersion);
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
index b4ca2d6..27733cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
@@ -35,6 +35,8 @@ public abstract class EditLogOutputStream implements Closeable {
// these are statistics counters
private long numSync; // number of sync(s) to disk
private long totalTimeSync; // total time to sync
+ // The version of the current edit log
+ private int currentLogVersion;
public EditLogOutputStream() throws IOException {
numSync = totalTimeSync = 0;
@@ -147,4 +149,18 @@ public abstract class EditLogOutputStream implements Closeable {
public String generateReport() {
return toString();
}
+
+ /**
+ * @return The version of the current edit log
+ */
+ public int getCurrentLogVersion() {
+ return currentLogVersion;
+ }
+
+ /**
+ * @param logVersion The version of the current edit log
+ */
+ public void setCurrentLogVersion(int logVersion) {
+ this.currentLogVersion = logVersion;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
index 4e1dab0..2603418 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
@@ -55,9 +55,9 @@ public class EditsDoubleBuffer {
bufReady = new TxnBuffer(initBufferSize);
}
-
- public void writeOp(FSEditLogOp op) throws IOException {
- bufCurrent.writeOp(op);
+
+ public void writeOp(FSEditLogOp op, int logVersion) throws IOException {
+ bufCurrent.writeOp(op, logVersion);
}
public void writeRaw(byte[] bytes, int offset, int length) throws IOException {
@@ -148,13 +148,13 @@ public class EditsDoubleBuffer {
reset();
}
- public void writeOp(FSEditLogOp op) throws IOException {
+ public void writeOp(FSEditLogOp op, int logVersion) throws IOException {
if (firstTxId == HdfsServerConstants.INVALID_TXID) {
firstTxId = op.txid;
} else {
assert op.txid > firstTxId;
}
- writer.writeOp(op);
+ writer.writeOp(op, logVersion);
numTxns++;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 2719d9b..600b924 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -155,7 +155,9 @@ public class FSEditLog implements LogsPurgeable {
//initialize
private JournalSet journalSet = null;
- private EditLogOutputStream editLogStream = null;
+
+ @VisibleForTesting
+ EditLogOutputStream editLogStream = null;
// a monotonically increasing counter that represents transactionIds.
// All of the threads which update/increment txid are synchronized,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 8293a82..ab189e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -282,6 +282,11 @@ public abstract class FSEditLogOp {
public abstract void writeFields(DataOutputStream out)
throws IOException;
+ public void writeFields(DataOutputStream out, int logVersion)
+ throws IOException {
+ writeFields(out);
+ }
+
static interface BlockListUpdatingOp {
Block[] getBlocks();
String getPath();
@@ -546,6 +551,12 @@ public abstract class FSEditLogOp {
@Override
public void writeFields(DataOutputStream out) throws IOException {
+ throw new IOException("Unsupported without logversion");
+ }
+
+ @Override
+ public void writeFields(DataOutputStream out, int logVersion)
+ throws IOException {
FSImageSerialization.writeLong(inodeId, out);
FSImageSerialization.writeString(path, out);
FSImageSerialization.writeShort(replication, out);
@@ -564,7 +575,10 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeString(clientMachine,out);
FSImageSerialization.writeBoolean(overwrite, out);
FSImageSerialization.writeByte(storagePolicyId, out);
- FSImageSerialization.writeByte(erasureCodingPolicyId, out);
+ if (NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.ERASURE_CODING, logVersion)) {
+ FSImageSerialization.writeByte(erasureCodingPolicyId, out);
+ }
// write clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
}
@@ -4854,16 +4868,18 @@ public abstract class FSEditLogOp {
* Write an operation to the output stream
*
* @param op The operation to write
+ * @param logVersion The version of edit log
* @throws IOException if an error occurs during writing.
*/
- public void writeOp(FSEditLogOp op) throws IOException {
+ public void writeOp(FSEditLogOp op, int logVersion)
+ throws IOException {
int start = buf.getLength();
// write the op code first to make padding and terminator verification
// work
buf.writeByte(op.opCode.getOpCode());
buf.writeInt(0); // write 0 for the length first
buf.writeLong(op.txid);
- op.writeFields(buf);
+ op.writeFields(buf, logVersion);
int end = buf.getLength();
// write the length back: content of the op + 4 bytes checksum - op_code
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0c8a257..42a8da1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2444,6 +2444,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
.getErasureCodingPolicy(this, ecPolicyName, iip);
if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
+ checkErasureCodingSupported("createWithEC");
if (blockSize < ecPolicy.getCellSize()) {
throw new IOException("Specified block size (" + blockSize
+ ") is less than the cell size (" + ecPolicy.getCellSize()
@@ -7485,6 +7486,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "setErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
@@ -7517,6 +7519,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final String operationName = "addErasureCodingPolicies";
List<String> addECPolicyNames = new ArrayList<>(policies.length);
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
List<AddErasureCodingPolicyResponse> responses =
new ArrayList<>(policies.length);
boolean success = false;
@@ -7558,6 +7561,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final boolean logRetryCache) throws IOException {
final String operationName = "removeErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
boolean success = false;
writeLock();
try {
@@ -7588,6 +7592,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final boolean logRetryCache) throws IOException {
final String operationName = "enableErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
boolean success = false;
writeLock();
try {
@@ -7619,6 +7624,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final boolean logRetryCache) throws IOException {
final String operationName = "disableErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
boolean success = false;
LOG.info("Disable the erasure coding policy " + ecPolicyName);
writeLock();
@@ -7652,6 +7658,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
UnresolvedLinkException, SafeModeException, AccessControlException {
final String operationName = "unsetErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
+ checkErasureCodingSupported(operationName);
FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
@@ -7679,6 +7686,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final String operationName = "getErasureCodingPolicy";
boolean success = false;
checkOperation(OperationCategory.READ);
+ checkErasureCodingSupported(operationName);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
@@ -7700,6 +7708,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final String operationName = "getErasureCodingPolicies";
boolean success = false;
checkOperation(OperationCategory.READ);
+ checkErasureCodingSupported(operationName);
readLock();
try {
checkOperation(OperationCategory.READ);
@@ -7720,6 +7729,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final String operationName = "getErasureCodingCodecs";
boolean success = false;
checkOperation(OperationCategory.READ);
+ checkErasureCodingSupported(operationName);
readLock();
try {
checkOperation(OperationCategory.READ);
@@ -8105,5 +8115,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return "disableRestoreFailedStorage";
}
}
+
+ /**
+ * Check whether operation is supported.
+ * @param operationName the name of operation.
+ * @throws UnsupportedActionException throws UAE if not supported.
+ */
+ public void checkErasureCodingSupported(String operationName)
+ throws UnsupportedActionException {
+ if (!NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.ERASURE_CODING,
+ getEffectiveLayoutVersion())) {
+ throw new UnsupportedActionException(operationName + " not supported.");
+ }
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/QJMTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/QJMTestUtil.java
index 5d939c8..9d95bfd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/QJMTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/QJMTestUtil.java
@@ -56,7 +56,7 @@ public abstract class QJMTestUtil {
for (long txid = startTxn; txid < startTxn + numTxns; txid++) {
FSEditLogOp op = NameNodeAdapter.createMkdirOp("tx " + txid);
op.setTransactionId(txid);
- writer.writeOp(op);
+ writer.writeOp(op, FAKE_NSINFO.getLayoutVersion());
}
return Arrays.copyOf(buf.getData(), buf.getLength());
@@ -73,7 +73,7 @@ public abstract class QJMTestUtil {
for (long txid = startTxId; txid < startTxId + numTxns; txid++) {
FSEditLogOp op = new TestEditLog.GarbageMkdirOp();
op.setTransactionId(txid);
- writer.writeOp(op);
+ writer.writeOp(op, FAKE_NSINFO.getLayoutVersion());
}
return Arrays.copyOf(buf.getData(), buf.getLength());
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index ce52221..b2524d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -64,13 +64,16 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -1706,4 +1709,77 @@ public class TestEditLog {
LogManager.getRootLogger().removeAppender(appender);
}
}
+
+ /**
+ * Test edits can be writen and read without ErasureCoding supported.
+ */
+ @Test
+ public void testEditLogWithoutErasureCodingSupported()
+ throws IOException {
+ Configuration conf = getConf();
+ MiniDFSCluster cluster = null;
+
+ // ERASURECODING not supported
+ int logVersion = -61;
+ assertFalse(NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.ERASURE_CODING, logVersion));
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+
+ final FSNamesystem namesystem = cluster.getNamesystem();
+ FSImage fsimage = namesystem.getFSImage();
+ FileSystem fileSys = cluster.getFileSystem();
+ final FSEditLog editLog = fsimage.getEditLog();
+ editLog.editLogStream.setCurrentLogVersion(logVersion);
+ // Write new version edit log
+ long txid = editLog.rollEditLog(logVersion);
+
+ String testDir = "/test";
+ String testFile = "testfile_001";
+ String testFilePath = testDir + "/" + testFile;
+
+ fileSys.mkdirs(new Path(testDir), new FsPermission("755"));
+
+ // Create a file
+ Path p = new Path(testFilePath);
+ DFSTestUtil.createFile(fileSys, p, 0, (short) 1, 1);
+
+ long blkId = 1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+ // Add a block to the file
+ BlockInfoContiguous blockInfo =
+ new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp),
+ (short)1);
+ INodeFile file
+ = (INodeFile)namesystem.getFSDirectory().getINode(testFilePath);
+ file.addBlock(blockInfo);
+ file.toUnderConstruction("testClient", "testMachine");
+
+ // Write edit log
+ editLog.logAddBlock(testFilePath, file);
+ editLog.rollEditLog(logVersion);
+
+ // Read edit log
+ Collection<EditLogInputStream> editStreams
+ = editLog.selectInputStreams(txid, txid + 1);
+ EditLogInputStream inputStream = null;
+ for (EditLogInputStream s : editStreams) {
+ if (s.getFirstTxId() == txid) {
+ inputStream = s;
+ break;
+ }
+ }
+ assertNotNull(inputStream);
+ int readLogVersion = inputStream.getVersion(false);
+ assertEquals(logVersion, readLogVersion);
+ FSEditLogLoader loader = new FSEditLogLoader(namesystem, 0);
+ long records = loader.loadFSEdits(inputStream, txid);
+ assertTrue(records > 0);
+
+ editLog.close();
+ cluster.shutdown();
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditsDoubleBuffer.java
index b75309e..3b15c2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditsDoubleBuffer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditsDoubleBuffer.java
@@ -87,6 +87,9 @@ public class TestEditsDoubleBuffer {
@Test
public void testDumpEdits() throws IOException {
final int defaultBufferSize = 256;
+ final int fakeLogVersion =
+ NameNodeLayoutVersion.Feature.ROLLING_UPGRADE
+ .getInfo().getLayoutVersion();
EditsDoubleBuffer buffer = new EditsDoubleBuffer(defaultBufferSize);
FSEditLogOp.OpInstanceCache cache = new FSEditLogOp.OpInstanceCache();
@@ -98,7 +101,7 @@ public class TestEditsDoubleBuffer {
.setPath(src)
.setReplication(replication);
op.setTransactionId(1);
- buffer.writeOp(op);
+ buffer.writeOp(op, fakeLogVersion);
src = "/testdumpedits2";
@@ -107,13 +110,13 @@ public class TestEditsDoubleBuffer {
.setPath(src)
.setTimestamp(0);
op2.setTransactionId(2);
- buffer.writeOp(op2);
+ buffer.writeOp(op2, fakeLogVersion);
FSEditLogOp.AllocateBlockIdOp op3 =
FSEditLogOp.AllocateBlockIdOp.getInstance(cache.get())
.setBlockId(0);
op3.setTransactionId(3);
- buffer.writeOp(op3);
+ buffer.writeOp(op3, fakeLogVersion);
GenericTestUtils.LogCapturer logs =
GenericTestUtils.LogCapturer.captureLogs(EditsDoubleBuffer.LOG);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org