You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/06/05 06:24:21 UTC

svn commit: r781907 - /hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java

Author: stack
Date: Fri Jun  5 04:24:21 2009
New Revision: 781907

URL: http://svn.apache.org/viewvc?rev=781907&view=rev
Log:
HBASE-1484 commit log split writes files with newest edits first (since hbase-1430); should be other way round

Modified:
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=781907&r1=781906&r2=781907&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Fri Jun  5 04:24:21 2009
@@ -24,6 +24,9 @@
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.util.ListIterator;
+import java.util.List;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Map;
@@ -93,7 +96,7 @@
  *
  */
 public class HLog implements HConstants, Syncable {
-  private static final Log LOG = LogFactory.getLog(HLog.class);
+  static final Log LOG = LogFactory.getLog(HLog.class);
   private static final String HLOG_DATFILE = "hlog.dat.";
   static final byte [] METACOLUMN = Bytes.toBytes("METACOLUMN:");
   static final byte [] METAROW = Bytes.toBytes("METAROW");
@@ -207,7 +210,7 @@
    * @param c Configuration to use.
    * @return the kind of compression to use
    */
-  private static CompressionType getCompressionType(final Configuration c) {
+  static CompressionType getCompressionType(final Configuration c) {
     String name = c.get("hbase.io.seqfile.compression.type");
     return name == null? CompressionType.NONE: CompressionType.valueOf(name);
   }
@@ -723,22 +726,23 @@
    * @param conf HBaseConfiguration
    * @throws IOException
    */
-  public static void splitLog(final Path rootDir, final Path srcDir,
+  public static List<Path> splitLog(final Path rootDir, final Path srcDir,
       final FileSystem fs, final Configuration conf)
   throws IOException {
     long millis = System.currentTimeMillis();
+    List<Path> splits = null;
     if (!fs.exists(srcDir)) {
       // Nothing to do
-      return;
+      return splits;
     }
     FileStatus [] logfiles = fs.listStatus(srcDir);
     if (logfiles == null || logfiles.length == 0) {
       // Nothing to do
-      return;
+      return splits;
     }
     LOG.info("Splitting " + logfiles.length + " log(s) in " +
       srcDir.toString());
-    splitLog(rootDir, logfiles, fs, conf);
+    splits = splitLog(rootDir, logfiles, fs, conf);
     try {
       fs.delete(srcDir, true);
     } catch (IOException e) {
@@ -750,6 +754,17 @@
     long endMillis = System.currentTimeMillis();
     LOG.info("log file splitting completed in " + (endMillis - millis) +
         " millis for " + srcDir.toString());
+    return splits;
+  }
+
+  // Private immutable datastructure to hold Writer and its Path.
+  private final static class WriterAndPath {
+    final Path p;
+    final SequenceFile.Writer w;
+    WriterAndPath(final Path p, final SequenceFile.Writer w) {
+      this.p = p;
+      this.w = w;
+    }
   }
   
   /*
@@ -759,12 +774,12 @@
    * @param conf
    * @throws IOException
    */
-  private static void splitLog(final Path rootDir, final FileStatus [] logfiles,
-      final FileSystem fs, final Configuration conf)
+  private static List<Path> splitLog(final Path rootDir,
+      final FileStatus [] logfiles, final FileSystem fs, final Configuration conf)
     throws IOException {
-      final Map<byte [], SequenceFile.Writer> logWriters =
-        new TreeMap<byte [], SequenceFile.Writer>(Bytes.BYTES_COMPARATOR);
-      
+      final Map<byte [], WriterAndPath> logWriters =
+        new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR);
+      List<Path> splits = null;
       try {
         int maxSteps = Double.valueOf(Math.ceil((logfiles.length * 1.0) / 
             DEFAULT_NUMBER_CONCURRENT_LOG_READS)).intValue();
@@ -849,9 +864,14 @@
                 long threadTime = System.currentTimeMillis();
                 try {
                   int count = 0;
-                  for (HLogEntry logEntry : entries) {
-                    SequenceFile.Writer w = logWriters.get(key);
-                    if (w == null) {
+                  // Items were added to the linkedlist oldest first. Pull them
+                  // out in that order.
+                  for (ListIterator<HLogEntry> i =
+                    entries.listIterator(entries.size());
+                      i.hasPrevious();) {
+                    HLogEntry logEntry = i.previous();
+                    WriterAndPath wap = logWriters.get(key);
+                    if (wap == null) {
                       Path logfile = new Path(HRegion.getRegionDir(HTableDescriptor
                           .getTableDir(rootDir, logEntry.getKey().getTablename()),
                           HRegionInfo.encodeRegionName(key)),
@@ -865,9 +885,10 @@
                         fs.rename(logfile, oldlogfile);
                         old = new SequenceFile.Reader(fs, oldlogfile, conf);
                       }
-                      w = SequenceFile.createWriter(fs, conf, logfile,
+                      SequenceFile.Writer w =
+                        SequenceFile.createWriter(fs, conf, logfile,
                           HLogKey.class, HLogEdit.class, getCompressionType(conf));
-                      logWriters.put(key, w);
+                      logWriters.put(key, new WriterAndPath(logfile, w));
                       if (LOG.isDebugEnabled()) {
                         LOG.debug("Creating new hlog file writer for path "
                             + logfile + " and region " + Bytes.toString(key));
@@ -888,7 +909,7 @@
                         fs.delete(oldlogfile, true);
                       }
                     }
-                    w.append(logEntry.getKey(), logEntry.getEdit());
+                    wap.w.append(logEntry.getKey(), logEntry.getEdit());
                     count++;
                   }
                   if (LOG.isDebugEnabled()) {
@@ -912,15 +933,18 @@
             for(int i = 0; !threadPool.awaitTermination(5, TimeUnit.SECONDS); i++) {
               LOG.debug("Waiting for hlog writers to terminate, iteration #" + i);
             }
-          }catch(InterruptedException ex) {
+          } catch(InterruptedException ex) {
             LOG.warn("Hlog writers were interrupted, possible data loss!");
           }
         }
       } finally {
-        for (SequenceFile.Writer w : logWriters.values()) {
-          w.close();
+        splits = new ArrayList<Path>(logWriters.size());
+        for (WriterAndPath wap: logWriters.values()) {
+          wap.w.close();
+          splits.add(wap.p);
         }
       }
+      return splits;
     }
   
   /**
@@ -955,6 +979,9 @@
       return key;
     }
 
+    public String toString() {
+      return this.key + "=" + this.edit;
+    }
   }
 
   /**