You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/04/28 08:00:56 UTC
svn commit: r1097329 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
Author: todd
Date: Thu Apr 28 06:00:55 2011
New Revision: 1097329
URL: http://svn.apache.org/viewvc?rev=1097329&view=rev
Log:
HDFS-1846. Preallocate edit log with OP_INVALID instead of zero bytes to ensure blocks are actually allocated. Contributed by Aaron T. Myers.
Added:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu Apr 28 06:00:55 2011
@@ -143,6 +143,9 @@ Trunk (unreleased changes)
HDFS-1862. Improve test reliability of HDFS-1594. (Aaron T. Myers via eli)
+ HDFS-1846. Preallocate edit log with OP_INVALID instead of zero bytes
+ to ensure blocks are actually allocated. (Aaron T. Myers via todd)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Thu Apr 28 06:00:55 2011
@@ -43,7 +43,14 @@ class EditLogFileOutputStream extends Ed
private DataOutputBuffer bufCurrent; // current buffer for writing
private DataOutputBuffer bufReady; // buffer ready for flushing
final private int initBufferSize; // inital buffer size
- static ByteBuffer fill = ByteBuffer.allocateDirect(512); // preallocation
+ static ByteBuffer fill = ByteBuffer.allocateDirect(1024 * 1024); // preallocation, 1MB
+
+ static {
+ fill.position(0);
+ for (int i = 0; i < fill.capacity(); i++) {
+ fill.put(FSEditLogOpCodes.OP_INVALID.getOpCode());
+ }
+ }
/**
* Creates output buffers and file object.
@@ -182,12 +189,11 @@ class EditLogFileOutputStream extends Ed
FSNamesystem.LOG.debug("Preallocating Edit log, current size "
+ fc.size());
}
- long newsize = position + 1024 * 1024; // 1MB
fill.position(0);
- int written = fc.write(fill, newsize);
+ int written = fc.write(fill, position);
if(FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug("Edit log size is now " + fc.size() +
- " written " + written + " bytes " + " at offset " + newsize);
+ " written " + written + " bytes " + " at offset " + position);
}
}
}
Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1097329&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java (added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java Thu Apr 28 06:00:55 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DU;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.Test;
+
+public class TestEditLogFileOutputStream {
+
+ @Test
+ public void testPreallocation() throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+ conf.set("dfs.http.address", "127.0.0.1:0");
+ NameNode.format(conf);
+ NameNode nn = new NameNode(conf);
+
+ File editLog = nn.getFSImage().getEditLog().getFsEditName();
+
+ assertEquals("Edit log should only be 4 bytes long",
+ 4, editLog.length());
+ assertEquals("Edit log disk space used should be one block",
+ 4096, new DU(editLog, conf).getUsed());
+
+ nn.mkdirs("/tmp", new FsPermission((short)777), false);
+
+ assertEquals("Edit log should be 1MB + 4 bytes long",
+ (1024 * 1024) + 4, editLog.length());
+ // 256 blocks for the 1MB of preallocation space, 1 block for the original
+ // 4 bytes
+ assertTrue("Edit log disk space used should be at least 257 blocks",
+ 257 * 4096 <= new DU(editLog, conf).getUsed());
+ }
+
+}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java Thu Apr 28 06:00:55 2011
@@ -238,11 +238,11 @@ public class TestOfflineEditsViewer {
// compares position to limit
if(!small.equals(large)) { return false; }
- // everything after limit should be zero
+ // everything after limit should be 0xFF
int i = large.limit();
large.clear();
for(; i < large.capacity(); i++) {
- if(large.get(i) != 0) {
+ if(large.get(i) != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
return false;
}
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
Binary files - no diff available.