You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2011/04/05 23:51:09 UTC
svn commit: r1089256 - in /hadoop/hdfs/branches/HDFS-1052: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/datanode/ sr...
Author: suresh
Date: Tue Apr 5 21:51:07 2011
New Revision: 1089256
URL: http://svn.apache.org/viewvc?rev=1089256&view=rev
Log:
Merging changes r1083084:r1083902 from trunk to federation
Modified:
hadoop/hdfs/branches/HDFS-1052/ (props changed)
hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
hadoop/hdfs/branches/HDFS-1052/build.xml (props changed)
hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/java/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/HDFS-1052/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036738,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036738,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Modified: hadoop/hdfs/branches/HDFS-1052/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/CHANGES.txt?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/CHANGES.txt (original)
+++ hadoop/hdfs/branches/HDFS-1052/CHANGES.txt Tue Apr 5 21:51:07 2011
@@ -307,6 +307,9 @@ Trunk (unreleased changes)
HDFS-1763. Replace hard-coded option strings with variables from
DFSConfigKeys. (eli)
+ HDFS-1521. Persist transaction ID on disk between NN restarts. (Ivan Kelly
+ and Todd Lipcon via todd)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
@@ -367,6 +370,8 @@ Trunk (unreleased changes)
HDFS-1757. Don't compile fuse-dfs by default. (eli)
+ HDFS-1770. TestFiRename fails due to invalid block size. (eli)
+
Release 0.22.0 - Unreleased
NEW FEATURES
Propchange: hadoop/hdfs/branches/HDFS-1052/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Propchange: hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Propchange: hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java Tue Apr 5 21:51:07 2011
@@ -88,7 +88,7 @@ public interface FSConstants {
// Version is reflected in the data storage file.
// Versions are negative.
// Decrement LAYOUT_VERSION to define a new version.
- public static final int LAYOUT_VERSION = -28;
+ public static final int LAYOUT_VERSION = -29;
// Current version:
- // -28: Adding support for block pools and multiple namenodes
+ // -29: Adding support for block pools and multiple namenodes
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Tue Apr 5 21:51:07 2011
@@ -79,7 +79,7 @@ public abstract class Storage extends St
public static final int PRE_RBW_LAYOUT_VERSION = -19;
// last layout version that is before federation
- public static final int LAST_PRE_FEDERATION_LAYOUT_VERSION = -27;
+ public static final int LAST_PRE_FEDERATION_LAYOUT_VERSION = -28;
private static final String STORAGE_FILE_LOCK = "in_use.lock";
protected static final String STORAGE_FILE_VERSION = "VERSION";
Propchange: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -4,4 +4,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java Tue Apr 5 21:51:07 2011
@@ -55,6 +55,7 @@ public class BackupImage extends FSImage
/** Is journal spooling in progress */
volatile JSpoolState jsState;
+ private long lastAppliedTxId = 0;
static enum JSpoolState {
OFF,
@@ -153,6 +154,11 @@ public class BackupImage extends FSImage
if(!editLog.isOpen())
editLog.open();
+ // set storage fields
+ storage.setStorageInfo(sig);
+ storage.setImageDigest(sig.getImageDigest());
+ storage.setCheckpointTime(sig.checkpointTime);
+
FSDirectory fsDir = getFSNamesystem().dir;
if(fsDir.isEmpty()) {
Iterator<StorageDirectory> itImage
@@ -163,6 +169,7 @@ public class BackupImage extends FSImage
throw new IOException("Could not locate checkpoint directories");
StorageDirectory sdName = itImage.next();
StorageDirectory sdEdits = itEdits.next();
+
getFSDirectoryRootLock().writeLock();
try { // load image under rootDir lock
loadFSImage(NNStorage.getStorageFile(sdName, NameNodeFile.IMAGE));
@@ -170,12 +177,8 @@ public class BackupImage extends FSImage
getFSDirectoryRootLock().writeUnlock();
}
loadFSEdits(sdEdits);
+ lastAppliedTxId = getEditLog().getLastWrittenTxId();
}
-
- // set storage fields
- storage.setStorageInfo(sig);
- storage.setImageDigest(sig.imageDigest);
- storage.setCheckpointTime(sig.checkpointTime);
}
/**
@@ -227,7 +230,8 @@ public class BackupImage extends FSImage
backupInputStream.setBytes(data);
FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
logLoader.loadEditRecords(storage.getLayoutVersion(),
- backupInputStream.getDataInputStream(), true);
+ backupInputStream.getDataInputStream(), true,
+ lastAppliedTxId + 1);
getFSNamesystem().dir.updateCountForINodeWithQuota(); // inefficient!
break;
case INPROGRESS:
@@ -348,12 +352,18 @@ public class BackupImage extends FSImage
EditLogFileInputStream edits = new EditLogFileInputStream(jSpoolFile);
DataInputStream in = edits.getDataInputStream();
FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
- numEdits += logLoader.loadFSEdits(in, false);
+ int loaded = logLoader.loadFSEdits(in, false, lastAppliedTxId + 1);
+
+ lastAppliedTxId += loaded;
+ numEdits += loaded;
// first time reached the end of spool
jsState = JSpoolState.WAIT;
- numEdits += logLoader.loadEditRecords(storage.getLayoutVersion(),
- in, true);
+ loaded = logLoader.loadEditRecords(storage.getLayoutVersion(),
+ in, true, lastAppliedTxId + 1);
+ numEdits += loaded;
+ lastAppliedTxId += loaded;
+
getFSNamesystem().dir.updateCountForINodeWithQuota();
edits.close();
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java Tue Apr 5 21:51:07 2011
@@ -51,15 +51,18 @@ class EditLogBackupOutputStream extends
static class JournalRecord {
byte op;
+ long txid;
Writable[] args;
- JournalRecord(byte op, Writable ... writables) {
+ JournalRecord(byte op, long txid, Writable ... writables) {
this.op = op;
+ this.txid = txid;
this.args = writables;
}
void write(DataOutputStream out) throws IOException {
out.write(op);
+ out.writeLong(txid);
if(args == null)
return;
for(Writable w : args)
@@ -105,8 +108,8 @@ class EditLogBackupOutputStream extends
}
@Override // EditLogOutputStream
- void write(byte op, Writable ... writables) throws IOException {
- bufCurrent.add(new JournalRecord(op, writables));
+ void write(byte op, long txid, Writable ... writables) throws IOException {
+ bufCurrent.add(new JournalRecord(op, txid, writables));
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Tue Apr 5 21:51:07 2011
@@ -81,10 +81,18 @@ class EditLogFileOutputStream extends Ed
bufCurrent.write(b);
}
- /** {@inheritDoc} */
+ /**
+ * Write a transaction to the stream. The serialization format is:
+ * <ul>
+ * <li>the opcode (byte)</li>
+ * <li>the transaction id (long)</li>
+ * <li>the actual Writables for the transaction</li>
+ * </ul>
+ * */
@Override
- void write(byte op, Writable... writables) throws IOException {
+ void write(byte op, long txid, Writable... writables) throws IOException {
write(op);
+ bufCurrent.writeLong(txid);
for (Writable w : writables) {
w.write(bufCurrent);
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java Tue Apr 5 21:51:07 2011
@@ -46,10 +46,12 @@ implements JournalStream {
* an array of Writable arguments.
*
* @param op operation
+ * @param txid the transaction ID of this operation
* @param writables array of Writable arguments
* @throws IOException
*/
- abstract void write(byte op, Writable ... writables) throws IOException;
+ abstract void write(byte op, long txid, Writable ... writables)
+ throws IOException;
/**
* Create and initialize underlying persistent edits log storage.
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Tue Apr 5 21:51:07 2011
@@ -306,12 +306,19 @@ public class FSEditLog implements NNStor
if(getNumEditStreams() == 0)
throw new java.lang.IllegalStateException(NO_JOURNAL_STREAMS_WARNING);
ArrayList<EditLogOutputStream> errorStreams = null;
+
+ // Only start a new transaction for OPs which will be persisted to disk.
+ // Obviously this excludes control op codes.
long start = now();
+ if (opCode.getOpCode() < FSEditLogOpCodes.OP_JSPOOL_START.getOpCode()) {
+ start = beginTransaction();
+ }
+
for(EditLogOutputStream eStream : editStreams) {
if(!eStream.isOperationSupported(opCode.getOpCode()))
continue;
try {
- eStream.write(opCode.getOpCode(), writables);
+ eStream.write(opCode.getOpCode(), txid, writables);
} catch (IOException ie) {
LOG.error("logEdit: removing "+ eStream.getName(), ie);
if(errorStreams == null)
@@ -320,7 +327,7 @@ public class FSEditLog implements NNStor
}
}
disableAndReportErrorOnStreams(errorStreams);
- recordTransaction(start);
+ endTransaction(start);
// check if it is time to schedule an automatic sync
if (!shouldForceSync()) {
@@ -371,7 +378,8 @@ public class FSEditLog implements NNStor
return false;
}
- private void recordTransaction(long start) {
+ private long beginTransaction() {
+ assert Thread.holdsLock(this);
// get a new transactionId
txid++;
@@ -380,7 +388,12 @@ public class FSEditLog implements NNStor
//
TransactionId id = myTransactionId.get();
id.txid = txid;
-
+ return now();
+ }
+
+ private void endTransaction(long start) {
+ assert Thread.holdsLock(this);
+
// update statistics
long end = now();
numTransactions++;
@@ -390,6 +403,21 @@ public class FSEditLog implements NNStor
}
/**
+ * Return the transaction ID of the last transaction written to the log.
+ */
+ synchronized long getLastWrittenTxId() {
+ return txid;
+ }
+
+ /**
+ * Set the transaction ID to use for the next transaction written.
+ */
+ synchronized void setNextTxId(long nextTxid) {
+ assert synctxid <= txid;
+ txid = nextTxid - 1;
+ }
+
+ /**
* Blocks until all ongoing edits have been synced to disk.
* This differs from logSync in that it waits for edits that have been
* written by other threads, not just edits from the calling thread.
@@ -784,6 +812,8 @@ public class FSEditLog implements NNStor
/**
* Closes the current edit log and opens edits.new.
+ * @return the transaction id that will be used as the first transaction
+ * in the new log
*/
synchronized void rollEditLog() throws IOException {
waitForSyncToFinish();
@@ -1014,7 +1044,7 @@ public class FSEditLog implements NNStor
if(getNumEditStreams() == 0)
throw new java.lang.IllegalStateException(NO_JOURNAL_STREAMS_WARNING);
ArrayList<EditLogOutputStream> errorStreams = null;
- long start = now();
+ long start = beginTransaction();
for(EditLogOutputStream eStream : editStreams) {
try {
eStream.write(data, 0, length);
@@ -1026,7 +1056,7 @@ public class FSEditLog implements NNStor
}
}
disableAndReportErrorOnStreams(errorStreams);
- recordTransaction(start);
+ endTransaction(start);
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Tue Apr 5 21:51:07 2011
@@ -53,17 +53,20 @@ public class FSEditLogLoader {
* This is where we apply edits that we've been writing to disk all
* along.
*/
- int loadFSEdits(EditLogInputStream edits) throws IOException {
+ int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId)
+ throws IOException {
DataInputStream in = edits.getDataInputStream();
long startTime = now();
- int numEdits = loadFSEdits(in, true);
+ int numEdits = loadFSEdits(in, true, expectedStartingTxId);
FSImage.LOG.info("Edits file " + edits.getName()
+ " of size " + edits.length() + " edits # " + numEdits
+ " loaded in " + (now()-startTime)/1000 + " seconds.");
return numEdits;
}
- int loadFSEdits(DataInputStream in, boolean closeOnExit) throws IOException {
+ int loadFSEdits(DataInputStream in, boolean closeOnExit,
+ long expectedStartingTxId)
+ throws IOException {
int numEdits = 0;
int logVersion = 0;
@@ -89,19 +92,19 @@ public class FSEditLogLoader {
}
assert logVersion <= Storage.LAST_UPGRADABLE_LAYOUT_VERSION :
"Unsupported version " + logVersion;
- numEdits = loadEditRecords(logVersion, in, false);
+
+ numEdits = loadEditRecords(logVersion, in, false, expectedStartingTxId);
} finally {
if(closeOnExit)
in.close();
}
- if (logVersion != FSConstants.LAYOUT_VERSION) // other version
- numEdits++; // save this image asap
+
return numEdits;
}
@SuppressWarnings("deprecation")
int loadEditRecords(int logVersion, DataInputStream in,
- boolean closeOnExit) throws IOException {
+ boolean closeOnExit, long expectedStartingTxId) throws IOException {
FSDirectory fsDir = fsNamesys.dir;
int numEdits = 0;
String clientName = null;
@@ -116,6 +119,8 @@ public class FSEditLogLoader {
numOpUpdateMasterKey = 0, numOpOther = 0;
try {
+ long txId = expectedStartingTxId - 1;
+
while (true) {
long timestamp = 0;
long mtime = 0;
@@ -133,6 +138,17 @@ public class FSEditLogLoader {
} catch (EOFException e) {
break; // no more transactions
}
+
+ if (logVersion <= -28) {
+ // Read the txid
+ long thisTxId = in.readLong();
+ if (thisTxId != txId + 1) {
+ throw new IOException("Expected transaction ID " +
+ (txId + 1) + " but got " + thisTxId);
+ }
+ txId = thisTxId;
+ }
+
numEdits++;
switch (opCode) {
case OP_ADD:
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Apr 5 21:51:07 2011
@@ -167,7 +167,7 @@ public class FSImage implements NNStorag
storage.setUpgradeManager(ns.upgradeManager);
}
}
-
+
void setCheckpointDirectories(Collection<URI> dirs,
Collection<URI> editsDirs) {
checkpointDirs = dirs;
@@ -661,16 +661,22 @@ public class FSImage implements NNStorag
//
// Load in bits
//
+ latestEditsSD.read();
+ long editsVersion = storage.getLayoutVersion();
latestNameSD.read();
- needToSave |= loadFSImage(NNStorage.getStorageFile(latestNameSD,
- NameNodeFile.IMAGE));
+ long imageVersion = storage.getLayoutVersion();
+
+ loadFSImage(NNStorage.getStorageFile(latestNameSD, NameNodeFile.IMAGE));
// Load latest edits
- if (latestNameCheckpointTime > latestEditsCheckpointTime)
+ if (latestNameCheckpointTime > latestEditsCheckpointTime) {
// the image is already current, discard edits
needToSave |= true;
- else // latestNameCheckpointTime == latestEditsCheckpointTime
- needToSave |= (loadFSEdits(latestEditsSD) > 0);
+ } else { // latestNameCheckpointTime == latestEditsCheckpointTime
+ needToSave |= loadFSEdits(latestEditsSD);
+ }
+ needToSave |= (editsVersion != FSConstants.LAYOUT_VERSION
+ || imageVersion != FSConstants.LAYOUT_VERSION);
return needToSave;
}
@@ -680,7 +686,7 @@ public class FSImage implements NNStorag
* filenames and blocks. Return whether we should
* "re-save" and consolidate the edit-logs
*/
- boolean loadFSImage(File curFile) throws IOException {
+ void loadFSImage(File curFile) throws IOException {
FSImageFormat.Loader loader = new FSImageFormat.Loader(
conf, getFSNamesystem());
loader.load(curFile);
@@ -697,44 +703,42 @@ public class FSImage implements NNStorag
" is corrupt with MD5 checksum of " + readImageMd5 +
" but expecting " + storage.getImageDigest());
}
-
- storage.namespaceID = loader.getLoadedNamespaceID();
- storage.layoutVersion = loader.getLoadedImageVersion();
-
- boolean needToSave =
- loader.getLoadedImageVersion() != FSConstants.LAYOUT_VERSION;
- return needToSave;
+ storage.setCheckpointTxId(loader.getLoadedImageTxId());
}
/**
* Load and merge edits from two edits files
*
* @param sd storage directory
- * @return number of edits loaded
+ * @return true if the image should be re-saved
* @throws IOException
*/
- int loadFSEdits(StorageDirectory sd) throws IOException {
+ boolean loadFSEdits(StorageDirectory sd) throws IOException {
FSEditLogLoader loader = new FSEditLogLoader(namesystem);
- int numEdits = 0;
EditLogFileInputStream edits =
new EditLogFileInputStream(NNStorage.getStorageFile(sd,
NameNodeFile.EDITS));
-
- numEdits = loader.loadFSEdits(edits);
+ long startingTxId = storage.getCheckpointTxId() + 1;
+ long numLoaded = loader.loadFSEdits(edits, startingTxId);
+ startingTxId += numLoaded;
+
edits.close();
File editsNew = NNStorage.getStorageFile(sd, NameNodeFile.EDITS_NEW);
if (editsNew.exists() && editsNew.length() > 0) {
edits = new EditLogFileInputStream(editsNew);
- numEdits += loader.loadFSEdits(edits);
+ numLoaded += loader.loadFSEdits(edits, startingTxId);
edits.close();
}
// update the counts.
getFSNamesystem().dir.updateCountForINodeWithQuota();
- return numEdits;
+ // update the txid for the edit log
+ editLog.setNextTxId(storage.getCheckpointTxId() + numLoaded + 1);
+
+ return numLoaded > 0;
}
/**
@@ -745,6 +749,7 @@ public class FSImage implements NNStorag
FSImageCompression compression = FSImageCompression.createCompression(conf);
saver.save(newFile, getFSNamesystem(), compression);
storage.setImageDigest(saver.getSavedDigest());
+ storage.setCheckpointTxId(editLog.getLastWrittenTxId());
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Tue Apr 5 21:51:07 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.permission.P
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
@@ -67,10 +68,8 @@ class FSImageFormat {
/** Set to true once a file has been loaded using this loader. */
private boolean loaded = false;
- /** The image version of the loaded file */
- private int imgVersion;
- /** The namespace ID of the loaded file */
- private int imgNamespaceID;
+ /** The transaction ID of the last edit represented by the loaded file */
+ private long imgTxId;
/** The MD5 sum of the loaded file */
private MD5Hash imgDigest;
@@ -80,15 +79,6 @@ class FSImageFormat {
}
/**
- * Return the version number of the image that has been loaded.
- * @throws IllegalStateException if load() has not yet been called.
- */
- int getLoadedImageVersion() {
- checkLoaded();
- return imgVersion;
- }
-
- /**
* Return the MD5 checksum of the image that has been loaded.
* @throws IllegalStateException if load() has not yet been called.
*/
@@ -97,13 +87,9 @@ class FSImageFormat {
return imgDigest;
}
- /**
- * Return the namespace ID of the image that has been loaded.
- * @throws IllegalStateException if load() has not yet been called.
- */
- int getLoadedNamespaceID() {
+ long getLoadedImageTxId() {
checkLoaded();
- return imgNamespaceID;
+ return imgTxId;
}
/**
@@ -152,10 +138,14 @@ class FSImageFormat {
* it should not contain version and namespace fields
*/
// read image version: first appeared in version -1
- imgVersion = in.readInt();
+ long imgVersion = in.readInt();
+ if(getLayoutVersion() != imgVersion)
+ throw new InconsistentFSStateException(curFile,
+ "imgVersion " + imgVersion +
+ " expected to be " + getLayoutVersion());
// read namespaceID: first appeared in version -2
- imgNamespaceID = in.readInt();
+ in.readInt();
// read number of files
long numFiles = readNumFiles(in);
@@ -165,6 +155,15 @@ class FSImageFormat {
long genstamp = in.readLong();
namesystem.setGenerationStamp(genstamp);
}
+
+ // read the transaction ID of the last edit represented by
+ // this image
+ if (imgVersion <= -28) {
+ imgTxId = in.readLong();
+ } else {
+ imgTxId = 0;
+ }
+
// read compression related info
FSImageCompression compression;
@@ -256,11 +255,12 @@ class FSImageFormat {
* @return an inode
*/
private INode loadINode(DataInputStream in)
- throws IOException {
+ throws IOException {
long modificationTime = 0;
long atime = 0;
long blockSize = 0;
+ long imgVersion = getLayoutVersion();
short replication = in.readShort();
replication = namesystem.adjustReplication(replication);
modificationTime = in.readLong();
@@ -326,7 +326,10 @@ class FSImageFormat {
modificationTime, atime, nsQuota, dsQuota, blockSize);
}
- private void loadDatanodes(DataInputStream in) throws IOException {
+ private void loadDatanodes(DataInputStream in)
+ throws IOException {
+ long imgVersion = getLayoutVersion();
+
if (imgVersion > -3) // pre datanode image version
return;
if (imgVersion <= -12) {
@@ -342,6 +345,7 @@ class FSImageFormat {
private void loadFilesUnderConstruction(DataInputStream in)
throws IOException {
FSDirectory fsDir = namesystem.dir;
+ long imgVersion = getLayoutVersion();
if (imgVersion > -13) // pre lease image version
return;
int size = in.readInt();
@@ -367,7 +371,10 @@ class FSImageFormat {
}
}
- private void loadSecretManagerState(DataInputStream in) throws IOException {
+ private void loadSecretManagerState(DataInputStream in)
+ throws IOException {
+ long imgVersion = getLayoutVersion();
+
if (imgVersion > -23) {
//SecretManagerState is not available.
//This must not happen if security is turned on.
@@ -376,8 +383,14 @@ class FSImageFormat {
namesystem.loadSecretManagerState(in);
}
+ private long getLayoutVersion() {
+ return namesystem.getFSImage().getStorage().getLayoutVersion();
+ }
+
+ private long readNumFiles(DataInputStream in)
+ throws IOException {
+ long imgVersion = getLayoutVersion();
- private long readNumFiles(DataInputStream in) throws IOException {
if (imgVersion <= -16) {
return in.readLong();
} else {
@@ -472,9 +485,12 @@ class FSImageFormat {
DataOutputStream out = new DataOutputStream(fos);
try {
out.writeInt(FSConstants.LAYOUT_VERSION);
- out.writeInt(sourceNamesystem.getFSImage().getStorage().getNamespaceID()); // TODO bad dependency
+ out.writeInt(sourceNamesystem.getFSImage()
+ .getStorage().getNamespaceID()); // TODO bad dependency
out.writeLong(fsDir.rootDir.numItemsInTree());
out.writeLong(sourceNamesystem.getGenerationStamp());
+ long txid = sourceNamesystem.getEditLog().getLastWrittenTxId();
+ out.writeLong(txid);
// write compression info and set up compressed stream
out = compression.writeHeaderAndWrapStream(fos);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Apr 5 21:51:07 2011
@@ -67,6 +67,7 @@ public class NNStorage extends Storage i
private static final Log LOG = LogFactory.getLog(NNStorage.class.getName());
static final String MESSAGE_DIGEST_PROPERTY = "imageMD5Digest";
+ static final String CHECKPOINT_TXID_PROPERTY = "checkpointTxId";
//
// The filenames used for storing the images
@@ -152,6 +153,13 @@ public class NNStorage extends Storage i
private long checkpointTime = -1L; // The age of the image
/**
+ * TxId of the last transaction that was included in the most
+ * recent fsimage file. This does not include any transactions
+ * that have since been written to the edit log.
+ */
+ protected long checkpointTxId;
+
+ /**
* list of failed (and thus removed) storages
*/
final protected List<StorageDirectory> removedStorageDirs
@@ -493,6 +501,20 @@ public class NNStorage extends Storage i
}
/**
+ * Set the transaction ID of the last checkpoint
+ */
+ void setCheckpointTxId(long checkpointTxId) {
+ this.checkpointTxId = checkpointTxId;
+ }
+
+ /**
+ * Return the transaction ID of the last checkpoint.
+ */
+ long getCheckpointTxId() {
+ return checkpointTxId;
+ }
+
+ /**
* Set the current checkpoint time. Writes the new checkpoint
* time to all available storage directories.
* @param newCpT The new checkpoint time.
@@ -701,6 +723,21 @@ public class NNStorage extends Storage i
" has image MD5 digest when version is " + layoutVersion);
}
+ String sCheckpointId = props.getProperty(CHECKPOINT_TXID_PROPERTY);
+ if (layoutVersion <= -28) {
+ if (sCheckpointId == null) {
+ throw new InconsistentFSStateException(sd.getRoot(),
+ "file " + STORAGE_FILE_VERSION
+ + " does not have the checkpoint transaction id set.");
+ }
+ this.checkpointTxId = Long.valueOf(sCheckpointId);
+ } else if (sCheckpointId != null) {
+ throw new InconsistentFSStateException(sd.getRoot(),
+ "file " + STORAGE_FILE_VERSION +
+ " has checkpoint transaction id when version is "
+ + layoutVersion);
+ }
+
this.setCheckpointTime(readCheckpointTime(sd));
}
@@ -736,7 +773,7 @@ public class NNStorage extends Storage i
}
props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString());
-
+ props.setProperty(CHECKPOINT_TXID_PROPERTY, String.valueOf(checkpointTxId));
writeCheckpointTime(sd);
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Apr 5 21:51:07 2011
@@ -711,9 +711,10 @@ public class SecondaryNameNode implement
sdEdits = it.next();
if (sdEdits == null)
throw new IOException("Could not locate checkpoint edits");
+
+ this.getStorage().setStorageInfo(sig);
+ this.getStorage().setImageDigest(sig.getImageDigest());
if (loadImage) {
- // to avoid assert in loadFSImage()
- this.getStorage().layoutVersion = -1;
loadFSImage(NNStorage.getStorageFile(sdName, NameNodeFile.IMAGE));
}
loadFSEdits(sdEdits);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java Tue Apr 5 21:51:07 2011
@@ -31,6 +31,7 @@ public enum EditsElement {
EDITS_VERSION,
RECORD,
OPCODE,
+ TRANSACTION_ID,
DATA,
// elements in the data part of the editLog records
LENGTH,
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Tue Apr 5 21:51:07 2011
@@ -77,7 +77,9 @@ class EditsLoaderCurrent implements Edit
* Visit OP_INVALID
*/
private void visit_OP_INVALID() throws IOException {
- ; // nothing to do, this op code has no data
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
}
/**
@@ -101,6 +103,9 @@ class EditsLoaderCurrent implements Edit
*/
private void visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes editsOpCode)
throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
IntToken opAddLength = v.visitInt(EditsElement.LENGTH);
// this happens if the edits is not properly ended (-1 op code),
@@ -144,6 +149,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_RENAME_OLD
*/
private void visit_OP_RENAME_OLD() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitInt( EditsElement.LENGTH);
v.visitStringUTF8( EditsElement.SOURCE);
v.visitStringUTF8( EditsElement.DESTINATION);
@@ -154,6 +163,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_DELETE
*/
private void visit_OP_DELETE() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitInt( EditsElement.LENGTH);
v.visitStringUTF8( EditsElement.PATH);
v.visitStringUTF8( EditsElement.TIMESTAMP);
@@ -163,6 +176,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_MKDIR
*/
private void visit_OP_MKDIR() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitInt( EditsElement.LENGTH);
v.visitStringUTF8( EditsElement.PATH);
v.visitStringUTF8( EditsElement.TIMESTAMP);
@@ -181,6 +198,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SET_REPLICATION
*/
private void visit_OP_SET_REPLICATION() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitStringUTF8(EditsElement.PATH);
v.visitStringUTF8(EditsElement.REPLICATION);
}
@@ -189,6 +210,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SET_PERMISSIONS
*/
private void visit_OP_SET_PERMISSIONS() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitStringUTF8( EditsElement.PATH);
v.visitShort( EditsElement.FS_PERMISSIONS);
}
@@ -197,6 +222,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SET_OWNER
*/
private void visit_OP_SET_OWNER() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitStringUTF8(EditsElement.PATH);
v.visitStringUTF8(EditsElement.USERNAME);
v.visitStringUTF8(EditsElement.GROUPNAME);
@@ -206,6 +235,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SET_GENSTAMP
*/
private void visit_OP_SET_GENSTAMP() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitLong(EditsElement.GENERATION_STAMP);
}
@@ -213,6 +246,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_TIMES
*/
private void visit_OP_TIMES() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitInt( EditsElement.LENGTH);
v.visitStringUTF8( EditsElement.PATH);
v.visitStringUTF8( EditsElement.MTIME);
@@ -223,6 +260,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SET_QUOTA
*/
private void visit_OP_SET_QUOTA() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitStringUTF8( EditsElement.PATH);
v.visitLong( EditsElement.NS_QUOTA);
v.visitLong( EditsElement.DS_QUOTA);
@@ -232,6 +273,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_RENAME
*/
private void visit_OP_RENAME() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
if(editsVersion > -21) {
throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_RENAME
+ " for edit log version " + editsVersion
@@ -248,6 +293,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_CONCAT_DELETE
*/
private void visit_OP_CONCAT_DELETE() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
if(editsVersion > -22) {
throw new IOException("Unexpected op code "
+ FSEditLogOpCodes.OP_CONCAT_DELETE
@@ -268,6 +317,10 @@ class EditsLoaderCurrent implements Edit
* Visit OP_SYMLINK
*/
private void visit_OP_SYMLINK() throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
v.visitInt( EditsElement.LENGTH);
v.visitStringUTF8( EditsElement.SOURCE);
v.visitStringUTF8( EditsElement.DESTINATION);
@@ -287,21 +340,25 @@ class EditsLoaderCurrent implements Edit
* Visit OP_GET_DELEGATION_TOKEN
*/
private void visit_OP_GET_DELEGATION_TOKEN() throws IOException {
- if(editsVersion > -24) {
- throw new IOException("Unexpected op code "
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ FSEditLogOpCodes.OP_GET_DELEGATION_TOKEN
+ " for edit log version " + editsVersion
+ " (op code 18 only expected for 24 and later)");
- }
- v.visitByte( EditsElement.T_VERSION);
- v.visitStringText( EditsElement.T_OWNER);
- v.visitStringText( EditsElement.T_RENEWER);
- v.visitStringText( EditsElement.T_REAL_USER);
- v.visitVLong( EditsElement.T_ISSUE_DATE);
- v.visitVLong( EditsElement.T_MAX_DATE);
- v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
- v.visitVInt( EditsElement.T_MASTER_KEY_ID);
- v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
}
/**
@@ -309,22 +366,25 @@ class EditsLoaderCurrent implements Edit
*/
private void visit_OP_RENEW_DELEGATION_TOKEN()
throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
- if(editsVersion > -24) {
- throw new IOException("Unexpected op code "
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ FSEditLogOpCodes.OP_RENEW_DELEGATION_TOKEN
+ " for edit log version " + editsVersion
+ " (op code 19 only expected for 24 and later)");
- }
- v.visitByte( EditsElement.T_VERSION);
- v.visitStringText( EditsElement.T_OWNER);
- v.visitStringText( EditsElement.T_RENEWER);
- v.visitStringText( EditsElement.T_REAL_USER);
- v.visitVLong( EditsElement.T_ISSUE_DATE);
- v.visitVLong( EditsElement.T_MAX_DATE);
- v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
- v.visitVInt( EditsElement.T_MASTER_KEY_ID);
- v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
}
/**
@@ -332,21 +392,24 @@ class EditsLoaderCurrent implements Edit
*/
private void visit_OP_CANCEL_DELEGATION_TOKEN()
throws IOException {
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
- if(editsVersion > -24) {
- throw new IOException("Unexpected op code "
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ FSEditLogOpCodes.OP_CANCEL_DELEGATION_TOKEN
+ " for edit log version " + editsVersion
+ " (op code 20 only expected for 24 and later)");
- }
- v.visitByte( EditsElement.T_VERSION);
- v.visitStringText( EditsElement.T_OWNER);
- v.visitStringText( EditsElement.T_RENEWER);
- v.visitStringText( EditsElement.T_REAL_USER);
- v.visitVLong( EditsElement.T_ISSUE_DATE);
- v.visitVLong( EditsElement.T_MAX_DATE);
- v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
- v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
}
/**
@@ -354,17 +417,20 @@ class EditsLoaderCurrent implements Edit
*/
private void visit_OP_UPDATE_MASTER_KEY()
throws IOException {
-
- if(editsVersion > -24) {
- throw new IOException("Unexpected op code "
+ if(editsVersion <= -28) {
+ v.visitLong(EditsElement.TRANSACTION_ID);
+ }
+
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ FSEditLogOpCodes.OP_UPDATE_MASTER_KEY
+ " for edit log version " + editsVersion
+ "(op code 21 only expected for 24 and later)");
- }
- v.visitVInt( EditsElement.KEY_ID);
- v.visitVLong( EditsElement.KEY_EXPIRY_DATE);
- VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
- v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
+ }
+ v.visitVInt( EditsElement.KEY_ID);
+ v.visitVLong( EditsElement.KEY_EXPIRY_DATE);
+ VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
+ v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
}
private void visitOpCode(FSEditLogOpCodes editsOpCode)
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Tue Apr 5 21:51:07 2011
@@ -156,6 +156,10 @@ class ImageLoaderCurrent implements Imag
v.visit(ImageElement.GENERATION_STAMP, in.readLong());
+ if (imageVersion <= -28) {
+ v.visit(ImageElement.TRANSACTION_ID, in.readLong());
+ }
+
if (imageVersion <= -25) {
boolean isCompressed = in.readBoolean();
v.visit(ImageElement.IS_COMPRESSED, imageVersion);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java Tue Apr 5 21:51:07 2011
@@ -71,7 +71,8 @@ abstract class ImageVisitor {
NUM_DELEGATION_TOKENS,
DELEGATION_TOKENS,
DELEGATION_TOKEN_IDENTIFIER,
- DELEGATION_TOKEN_EXPIRY_TIME
+ DELEGATION_TOKEN_EXPIRY_TIME,
+ TRANSACTION_ID
}
/**
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/aop/org/apache/hadoop/fs/TestFiRename.java Tue Apr 5 21:51:07 2011
@@ -24,6 +24,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.Rename;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.junit.After;
@@ -60,7 +61,6 @@ public class TestFiRename {
private static Configuration CONF = new Configuration();
static {
- CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
CONF.setInt("io.bytes.per.checksum", 1);
}
Propchange: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java Tue Apr 5 21:51:07 2011
@@ -171,6 +171,14 @@ public class TestBackupNode extends Test
//
backup = startBackupNode(conf, op, 1);
waitCheckpointDone(backup);
+
+ for (int i = 0; i < 10; i++) {
+ writeFile(fileSys, new Path("file_" + i), replication);
+ }
+
+ backup.doCheckpoint();
+ waitCheckpointDone(backup);
+
} catch(IOException e) {
LOG.error("Error in TestBackupNode:", e);
assertTrue(e.getLocalizedMessage(), false);
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Tue Apr 5 21:51:07 2011
@@ -30,12 +30,15 @@ import org.apache.hadoop.fs.permission.*
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
+
+import org.apache.hadoop.util.StringUtils;
import org.mockito.Mockito;
@@ -50,6 +53,26 @@ public class TestEditLog extends TestCas
static final int NUM_TRANSACTIONS = 100;
static final int NUM_THREADS = 100;
+ /** An edits log with 3 edits from 0.20 - the result of
+ * a fresh namesystem followed by hadoop fs -touchz /myfile */
+ static final byte[] HADOOP20_SOME_EDITS =
+ StringUtils.hexStringToByte((
+ "ffff ffed 0a00 0000 0000 03fa e100 0000" +
+ "0005 0007 2f6d 7966 696c 6500 0133 000d" +
+ "3132 3932 3331 3634 3034 3138 3400 0d31" +
+ "3239 3233 3136 3430 3431 3834 0009 3133" +
+ "3432 3137 3732 3800 0000 0004 746f 6464" +
+ "0a73 7570 6572 6772 6f75 7001 a400 1544" +
+ "4653 436c 6965 6e74 5f2d 3136 3136 3535" +
+ "3738 3931 000b 3137 322e 3239 2e35 2e33" +
+ "3209 0000 0005 0007 2f6d 7966 696c 6500" +
+ "0133 000d 3132 3932 3331 3634 3034 3138" +
+ "3400 0d31 3239 3233 3136 3430 3431 3834" +
+ "0009 3133 3432 3137 3732 3800 0000 0004" +
+ "746f 6464 0a73 7570 6572 6772 6f75 7001" +
+ "a4ff 0000 0000 0000 0000 0000 0000 0000"
+ ).replace(" ",""));
+
//
// an object that does a bunch of transactions
//
@@ -81,6 +104,51 @@ public class TestEditLog extends TestCas
}
/**
+ * Test case for an empty edit log from a prior version of Hadoop.
+ */
+ public void testPreTxIdEditLogNoEdits() throws Exception {
+ FSNamesystem namesys = Mockito.mock(FSNamesystem.class);
+ int numEdits = testLoad(
+ StringUtils.hexStringToByte("ffffffed"), // just version number
+ namesys);
+ assertEquals(0, numEdits);
+ }
+
+ /**
+ * Test case for loading a very simple edit log from a format
+ * prior to the inclusion of edit transaction IDs in the log.
+ */
+ public void testPreTxidEditLogWithEdits() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final FSNamesystem namesystem = cluster.getNamesystem();
+
+ int numEdits = testLoad(HADOOP20_SOME_EDITS, namesystem);
+ assertEquals(3, numEdits);
+ // Sanity check the edit
+ HdfsFileStatus fileInfo = namesystem.getFileInfo("/myfile", false);
+ assertEquals("supergroup", fileInfo.getGroup());
+ assertEquals(3, fileInfo.getReplication());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ private int testLoad(byte[] data, FSNamesystem namesys) throws IOException {
+ FSEditLogLoader loader = new FSEditLogLoader(namesys);
+ EditLogInputStream mockStream = Mockito.mock(EditLogInputStream.class);
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ Mockito.doReturn(new DataInputStream(bais))
+ .when(mockStream).getDataInputStream();
+
+ return loader.loadFSEdits(mockStream, 1);
+ }
+
+ /**
* Tests transaction logging in dfs.
*/
public void testEditLog() throws IOException {
@@ -145,13 +213,13 @@ public class TestEditLog extends TestCas
// If there were any corruptions, it is likely that the reading in
// of these transactions will throw an exception.
//
- FSEditLogLoader loader = new FSEditLogLoader(namesystem);
for (Iterator<StorageDirectory> it =
fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
+ FSEditLogLoader loader = new FSEditLogLoader(namesystem);
File editFile = NNStorage.getStorageFile(it.next(), NameNodeFile.EDITS);
System.out.println("Verifying file: " + editFile);
int numEdits = loader.loadFSEdits(
- new EditLogFileInputStream(editFile));
+ new EditLogFileInputStream(editFile), 1);
int numLeases = namesystem.leaseManager.countLease();
System.out.println("Number of outstanding leases " + numLeases);
assertEquals(0, numLeases);
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java Tue Apr 5 21:51:07 2011
@@ -187,20 +187,27 @@ public class TestEditLogRace {
startTransactionWorkers(namesystem, caughtErr);
+ long previousLogTxId = 1;
+
for (int i = 0; i < NUM_ROLLS && caughtErr.get() == null; i++) {
try {
Thread.sleep(20);
} catch (InterruptedException e) {}
LOG.info("Starting roll " + i + ".");
- editLog.rollEditLog();
- LOG.info("Roll complete " + i + ".");
+ long nextTxId = 0L;
+ synchronized (editLog) {
+ editLog.rollEditLog();
+ nextTxId = editLog.getLastWrittenTxId() + 1;
+ }
+ LOG.info("Roll " + i + " complete before txid " + nextTxId);
- verifyEditLogs(namesystem, fsimage);
+ verifyEditLogs(namesystem, fsimage, previousLogTxId);
LOG.info("Starting purge " + i + ".");
editLog.purgeEditLog();
LOG.info("Complete purge " + i + ".");
+ previousLogTxId = nextTxId;
}
} finally {
stopTransactionWorkers();
@@ -213,7 +220,8 @@ public class TestEditLogRace {
}
}
- private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage)
+ private void verifyEditLogs(FSNamesystem namesystem, FSImage fsimage,
+ long startingTxId)
throws IOException {
// Verify that we can read in all the transactions that we have written.
// If there were any corruptions, it is likely that the reading in
@@ -222,8 +230,9 @@ public class TestEditLogRace {
fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
File editFile = fsimage.getStorage().getStorageFile(it.next(), NameNodeFile.EDITS);
System.out.println("Verifying file: " + editFile);
- int numEdits = new FSEditLogLoader(namesystem).loadFSEdits(
- new EditLogFileInputStream(editFile));
+ FSEditLogLoader loader = new FSEditLogLoader(namesystem);
+ int numEdits = loader.loadFSEdits(new EditLogFileInputStream(editFile),
+ startingTxId);
System.out.println("Number of edits: " + numEdits);
}
}
@@ -265,14 +274,19 @@ public class TestEditLogRace {
namesystem.enterSafeMode();
// Verify edit logs before the save
- verifyEditLogs(namesystem, fsimage);
+ // They should start with the first edit after the checkpoint
+ verifyEditLogs(namesystem, fsimage,
+ fsimage.getStorage().getCheckpointTxId() + 1);
LOG.info("Save " + i + ": saving namespace");
namesystem.saveNamespace();
LOG.info("Save " + i + ": leaving safemode");
// Verify that edit logs post save are also not corrupt
- verifyEditLogs(namesystem, fsimage);
+ verifyEditLogs(namesystem, fsimage,
+ fsimage.getStorage().getCheckpointTxId() + 1);
+ assertEquals(fsimage.getStorage().getCheckpointTxId(),
+ editLog.getLastWrittenTxId());
namesystem.leaveSafeMode(false);
LOG.info("Save " + i + ": complete");
@@ -393,7 +407,9 @@ public class TestEditLogRace {
doAnEditThread.join();
assertNull(deferredException.get());
- verifyEditLogs(namesystem, fsimage);
+ // Since we did one transaction before the saveNamespace, the new
+ // log starts at txid 2
+ verifyEditLogs(namesystem, fsimage, 2);
} finally {
LOG.info("Closing namesystem");
if(namesystem != null) namesystem.close();
@@ -478,7 +494,9 @@ public class TestEditLogRace {
doAnEditThread.join();
assertNull(deferredException.get());
- verifyEditLogs(namesystem, fsimage);
+ // Since we did one transaction before the saveNamespace, the new
+ // log starts at txid 2
+ verifyEditLogs(namesystem, fsimage, 2);
} finally {
LOG.info("Closing namesystem");
if(namesystem != null) namesystem.close();
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java Tue Apr 5 21:51:07 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyObject;
@@ -30,6 +31,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -41,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.F
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -57,6 +60,10 @@ import org.mockito.stubbing.Answer;
* </ol>
*/
public class TestSaveNamespace {
+ static {
+ ((Log4JLogger)FSImage.LOG).getLogger().setLevel(Level.ALL);
+ }
+
private static final Log LOG = LogFactory.getLog(TestSaveNamespace.class);
private static class FaultySaveImage implements Answer<Void> {
@@ -307,7 +314,37 @@ public class TestSaveNamespace {
}
}
}
+
+ @Test
+ public void testTxIdPersistence() throws Exception {
+ Configuration conf = getConf();
+ NameNode.initMetrics(conf, NamenodeRole.ACTIVE);
+ NameNode.format(conf);
+ FSNamesystem fsn = new FSNamesystem(conf);
+ try {
+ assertEquals(0, fsn.getEditLog().getLastWrittenTxId());
+ doAnEdit(fsn, 1);
+ assertEquals(1, fsn.getEditLog().getLastWrittenTxId());
+
+ fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fsn.saveNamespace();
+
+ // Shut down and restart
+ fsn.getFSImage().close();
+ fsn.close();
+ fsn = null;
+
+ fsn = new FSNamesystem(conf);
+ assertEquals(1, fsn.getEditLog().getLastWrittenTxId());
+
+ } finally {
+ if (fsn != null) {
+ fsn.close();
+ }
+ }
+ }
+
private void doAnEdit(FSNamesystem fsn, int id) throws IOException {
// Make an edit
fsn.mkdirs(
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java?rev=1089256&r1=1089255&r2=1089256&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Tue Apr 5 21:51:07 2011
@@ -134,15 +134,15 @@ public class TestSecurityTokenEditLog ex
// If there were any corruptions, it is likely that the reading in
// of these transactions will throw an exception.
//
- FSEditLogLoader loader = new FSEditLogLoader(namesystem);
namesystem.getDelegationTokenSecretManager().stopThreads();
int numKeys = namesystem.getDelegationTokenSecretManager().getNumberOfKeys();
for (Iterator<StorageDirectory> it =
fsimage.getStorage().dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
+ FSEditLogLoader loader = new FSEditLogLoader(namesystem);
File editFile = fsimage.getStorage().getStorageFile(it.next(), NameNodeFile.EDITS);
System.out.println("Verifying file: " + editFile);
int numEdits = loader.loadFSEdits(
- new EditLogFileInputStream(editFile));
+ new EditLogFileInputStream(editFile), 1);
assertTrue("Verification for " + editFile + " failed. " +
"Expected " + (NUM_THREADS * opsPerTrans * NUM_TRANSACTIONS + numKeys) + " transactions. "+
"Found " + numEdits + " transactions.",
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Apr 5 21:51:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1036767,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066305,1066970,1066986,1067079-1076024,1076696,1078925,1078940,1079069-1080836,1081580-1083021,1083043,1083084,1083234,1083902