You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/11/07 19:10:39 UTC

svn commit: r1539743 [1/2] - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/wal/ test/java/org/apache/hadoop/hbase/coprocessor/ test/java/org/apache/hadoop/hbase/mapreduc...

Author: stack
Date: Thu Nov  7 18:10:38 2013
New Revision: 1539743

URL: http://svn.apache.org/r1539743
Log:
HBASE-8741 Scope sequenceid to the region rather than regionserver (WAS: Mutations on Regions in recovery mode might have same sequenceIDs)

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Nov  7 18:10:38 2013
@@ -209,6 +209,13 @@ public class HRegion implements HeapSize
   protected long completeSequenceId = -1L;
 
   /**
+   * Region level sequence Id. It is used for appending WALEdits in HLog. Its default value is -1,
+   * as a marker that the region hasn't opened yet. Once it is opened, it is set to
+   * {@link #openSeqNum}.
+   */
+  private final AtomicLong sequenceId = new AtomicLong(-1L);
+
+  /**
    * Operation enum is used in {@link HRegion#startRegionOperation} to provide operation context for
    * startRegionOperation to possibly invoke different checks before any region operations. Not all
    * operations have to be defined here. It's only needed when a special check is need in
@@ -1518,16 +1525,16 @@ public class HRegion implements HeapSize
       // Record the mvcc for all transactions in progress.
       w = mvcc.beginMemstoreInsert();
       mvcc.advanceMemstore(w);
-
+      // check if it is not closing.
       if (wal != null) {
-        Long startSeqId = wal.startCacheFlush(this.getRegionInfo().getEncodedNameAsBytes());
-        if (startSeqId == null) {
-          status.setStatus("Flush will not be started for [" + this.getRegionInfo().getEncodedName()
-              + "] - WAL is going away");
+        if (!wal.startCacheFlush(this.getRegionInfo().getEncodedNameAsBytes())) {
+          status.setStatus("Flush will not be started for ["
+              + this.getRegionInfo().getEncodedName() + "] - because the WAL is closing.");
           return false;
         }
-        flushSeqId = startSeqId;
+        flushSeqId = this.sequenceId.incrementAndGet();
       } else {
+        // use the provided sequence Id as WAL is not being used for this flush.
         flushSeqId = myseqid;
       }
 
@@ -2221,7 +2228,7 @@ public class HRegion implements HeapSize
       Mutation mutation = batchOp.operations[firstIndex];
       if (walEdit.size() > 0) {
         txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getTableName(),
-              walEdit, mutation.getClusterIds(), now, this.htableDescriptor);
+              walEdit, mutation.getClusterIds(), now, this.htableDescriptor, this.sequenceId);
       }
 
       // -------------------------------
@@ -3312,7 +3319,7 @@ public class HRegion implements HeapSize
           if(bulkLoadListener != null) {
             finalPath = bulkLoadListener.prepareBulkLoad(familyName, path);
           }
-          store.bulkLoadHFile(finalPath, assignSeqId ? this.log.obtainSeqNum() : -1);
+          store.bulkLoadHFile(finalPath, assignSeqId ? this.sequenceId.incrementAndGet() : -1);
           if(bulkLoadListener != null) {
             bulkLoadListener.doneBulkLoad(familyName, path);
           }
@@ -3906,7 +3913,9 @@ public class HRegion implements HeapSize
     HRegion region = HRegion.newHRegion(tableDir,
         effectiveHLog, fs, conf, info, hTableDescriptor, null);
     if (initialize) {
-      region.initialize();
+      // If initializing, set the sequenceId. It is also required by HLogPerformanceEvaluation when
+      // verifying the WALEdits.
+      region.setSequenceId(region.initialize());
     }
     return region;
   }
@@ -4087,10 +4096,7 @@ public class HRegion implements HeapSize
     checkCompressionCodecs();
 
     this.openSeqNum = initialize(reporter);
-    if (this.log != null) {
-      this.log.setSequenceNumber(this.openSeqNum);
-    }
-
+    this.setSequenceId(openSeqNum);
     return this;
   }
 
@@ -4498,8 +4504,9 @@ public class HRegion implements HeapSize
           long txid = 0;
           // 7. Append no sync
           if (!walEdit.isEmpty()) {
-            txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getTableName(),
-                  walEdit, processor.getClusterIds(), now, this.htableDescriptor);
+            txid = this.log.appendNoSync(this.getRegionInfo(),
+              this.htableDescriptor.getTableName(), walEdit, processor.getClusterIds(), now,
+              this.htableDescriptor, this.sequenceId);
           }
           // 8. Release region lock
           if (locked) {
@@ -4740,9 +4747,9 @@ public class HRegion implements HeapSize
             // Using default cluster id, as this can only happen in the orginating
             // cluster. A slave cluster receives the final value (not the delta)
             // as a Put.
-            txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getTableName(),
-              walEdits, new ArrayList<UUID>(), EnvironmentEdgeManager.currentTimeMillis(),
-                  this.htableDescriptor);
+            txid = this.log.appendNoSync(this.getRegionInfo(),
+              this.htableDescriptor.getTableName(), walEdits, new ArrayList<UUID>(),
+              EnvironmentEdgeManager.currentTimeMillis(), this.htableDescriptor, this.sequenceId);
           } else {
             recordMutationWithoutWal(append.getFamilyCellMap());
           }
@@ -4914,9 +4921,9 @@ public class HRegion implements HeapSize
             // Using default cluster id, as this can only happen in the orginating
             // cluster. A slave cluster receives the final value (not the delta)
             // as a Put.
-            txid = this.log.appendNoSync(this.getRegionInfo(), this.htableDescriptor.getTableName(),
-                walEdits, new ArrayList<UUID>(), EnvironmentEdgeManager.currentTimeMillis(),
-                  this.htableDescriptor);
+            txid = this.log.appendNoSync(this.getRegionInfo(),
+              this.htableDescriptor.getTableName(), walEdits, new ArrayList<UUID>(),
+              EnvironmentEdgeManager.currentTimeMillis(), this.htableDescriptor, this.sequenceId);
           } else {
             recordMutationWithoutWal(increment.getFamilyCellMap());
           }
@@ -4981,7 +4988,7 @@ public class HRegion implements HeapSize
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      40 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
+      41 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
       (11 * Bytes.SIZEOF_LONG) +
       5 * Bytes.SIZEOF_BOOLEAN);
 
@@ -5566,6 +5573,21 @@ public class HRegion implements HeapSize
   }
 
   /**
+   * @return sequenceId.
+   */
+  public AtomicLong getSequenceId() {
+    return this.sequenceId;
+  }
+
+  /**
+   * sets this region's sequenceId.
+   * @param value new value
+   */
+  private void setSequenceId(long value) {
+    this.sequenceId.set(value);
+  }
+
+  /**
    * Listener class to enable callers of
    * bulkLoadHFile() to perform any necessary
    * pre/post processing of a given bulkload call

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Thu Nov  7 18:10:38 2013
@@ -1074,7 +1074,7 @@ public class HStore implements Store {
     CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
         family.getName(), inputPaths, outputPaths, fs.getStoreDir(getFamily().getNameAsString()));
     HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
-        this.region.getRegionInfo(), compactionDescriptor);
+        this.region.getRegionInfo(), compactionDescriptor, this.region.getSequenceId());
   }
 
   private void replaceStoreFiles(final Collection<StoreFile> compactedFiles,

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java Thu Nov  7 18:10:38 2013
@@ -26,13 +26,13 @@ import java.lang.reflect.Method;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.SortedMap;
+import java.util.NavigableMap;
 import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -143,13 +143,6 @@ class FSHLog implements HLog, Syncable {
   Writer writer;
 
   /**
-   * Map of all log files but the current one.
-   */
-  final SortedMap<Long, Path> outputfiles =
-    Collections.synchronizedSortedMap(new TreeMap<Long, Path>());
-
-
-  /**
    * This lock synchronizes all operations on oldestUnflushedSeqNums and oldestFlushingSeqNums,
    * with the exception of append's putIfAbsent into oldestUnflushedSeqNums.
    * We only use these to find out the low bound seqNum, or to find regions with old seqNums to
@@ -177,8 +170,6 @@ class FSHLog implements HLog, Syncable {
 
   private volatile boolean closed = false;
 
-  private final AtomicLong logSeqNum = new AtomicLong(0);
-
   private boolean forMeta = false;
 
   // The timestamp (in ms) when the log file was created.
@@ -228,6 +219,39 @@ class FSHLog implements HLog, Syncable {
 
   private final AtomicInteger closeErrorCount = new AtomicInteger();
   private final MetricsWAL metrics;
+/**
+ * Map of region encoded names to the latest sequence num obtained from them while appending
+ * WALEdits to the wal. We create one map for each WAL file at the time it is rolled.
+ * <p>
+ * When deciding whether to archive a WAL file, we compare the sequence IDs in this map to
+ * {@link #oldestFlushingSeqNums} and {@link #oldestUnflushedSeqNums}.
+ * See {@link FSHLog#areAllRegionsFlushed(Map, Map, Map)} for more info.
+ * <p>
+ * This map uses byte[] as the key, and uses reference equality. It works in our use case as we
+ * use {@link HRegionInfo#getEncodedNameAsBytes()} as keys. For a given region, it always returns
+ * the same array.
+ */
+  private Map<byte[], Long> latestSequenceNums = new HashMap<byte[], Long>();
+
+  /**
+   * WAL Comparator; it compares the timestamp (log filenum), present in the log file name.
+   */
+  public final Comparator<Path> LOG_NAME_COMPARATOR = new Comparator<Path>() {
+    @Override
+    public int compare(Path o1, Path o2) {
+      long t1 = getFileNumFromFileName(o1);
+      long t2 = getFileNumFromFileName(o2);
+      if (t1 == t2) return 0;
+      return (t1 > t2) ? 1 : -1;
+    }
+  };
+
+  /**
+   * Map of log file to the latest sequence nums of all regions it has entries of.
+   * The map is sorted by the log file creation timestamp (contained in the log file name).
+   */
+  private NavigableMap<Path, Map<byte[], Long>> hlogSequenceNums =
+    new ConcurrentSkipListMap<Path, Map<byte[], Long>>(LOG_NAME_COMPARATOR);
 
   /**
    * Constructor.
@@ -436,21 +460,6 @@ class FSHLog implements HLog, Syncable {
     return this.filenum;
   }
 
-  @Override
-  public void setSequenceNumber(final long newvalue) {
-    for (long id = this.logSeqNum.get(); id < newvalue &&
-        !this.logSeqNum.compareAndSet(id, newvalue); id = this.logSeqNum.get()) {
-      // This could spin on occasion but better the occasional spin than locking
-      // every increment of sequence number.
-      LOG.debug("Changed sequenceid from " + id + " to " + newvalue);
-    }
-  }
-
-  @Override
-  public long getSequenceNumber() {
-    return logSeqNum.get();
-  }
-
   /**
    * Method used internal to this class and for tests only.
    * @return The wrapped stream our writer is using; its not the
@@ -524,11 +533,17 @@ class FSHLog implements HLog, Syncable {
           this.writer = nextWriter;
           this.hdfs_out = nextHdfsOut;
           this.numEntries.set(0);
+          if (oldFile != null) {
+            this.hlogSequenceNums.put(oldFile, this.latestSequenceNums);
+            this.latestSequenceNums = new HashMap<byte[], Long>();
+          }
         }
         if (oldFile == null) LOG.info("New WAL " + FSUtils.getPath(newPath));
-        else LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries=" + oldNumEntries +
+        else {
+          LOG.info("Rolled WAL " + FSUtils.getPath(oldFile) + " with entries=" + oldNumEntries +
           ", filesize=" + StringUtils.humanReadableInt(this.fs.getFileStatus(oldFile).getLen()) +
           "; new WAL " + FSUtils.getPath(newPath));
+        }
 
         // Tell our listeners that a new log was created
         if (!this.listeners.isEmpty()) {
@@ -540,7 +555,7 @@ class FSHLog implements HLog, Syncable {
         // Can we delete any of the old log files?
         if (getNumLogFiles() > 0) {
           cleanOldLogs();
-          regionsToFlush = getRegionsToForceFlush();
+          regionsToFlush = findRegionsToForceFlush();
         }
       } finally {
         this.logRollRunning = false;
@@ -568,87 +583,126 @@ class FSHLog implements HLog, Syncable {
     return HLogFactory.createWALWriter(fs, path, conf);
   }
 
-  /*
-   * Clean up old commit logs.
-   * @return If lots of logs, flush the returned region so next time through
-   * we can clean logs. Returns null if nothing to flush.  Returns array of
-   * encoded region names to flush.
+  /**
+   * Archive old logs that could be archived: a log is eligible for archiving if all its WALEdits
+   * are already flushed by the corresponding regions.
+   * <p>
+   * For each log file, it compares its region to sequenceId map
+   * (@link {@link FSHLog#latestSequenceNums} with corresponding region entries in
+   * {@link FSHLog#oldestFlushingSeqNums} and {@link FSHLog#oldestUnflushedSeqNums}.
+   * If all the regions in the map are flushed past of their value, then the wal is eligible for
+   * archiving.
    * @throws IOException
    */
   private void cleanOldLogs() throws IOException {
-    long oldestOutstandingSeqNum = Long.MAX_VALUE;
+    Map<byte[], Long> oldestFlushingSeqNumsLocal = null;
+    Map<byte[], Long> oldestUnflushedSeqNumsLocal = null;
+    List<Path> logsToArchive = new ArrayList<Path>();
+    // make a local copy so as to avoid locking when we iterate over these maps.
     synchronized (oldestSeqNumsLock) {
-      Long oldestFlushing = (oldestFlushingSeqNums.size() > 0)
-        ? Collections.min(oldestFlushingSeqNums.values()) : Long.MAX_VALUE;
-      Long oldestUnflushed = (oldestUnflushedSeqNums.size() > 0)
-        ? Collections.min(oldestUnflushedSeqNums.values()) : Long.MAX_VALUE;
-      oldestOutstandingSeqNum = Math.min(oldestFlushing, oldestUnflushed);
+      oldestFlushingSeqNumsLocal = new HashMap<byte[], Long>(this.oldestFlushingSeqNums);
+      oldestUnflushedSeqNumsLocal = new HashMap<byte[], Long>(this.oldestUnflushedSeqNums);
     }
-
-    // Get the set of all log files whose last sequence number is smaller than
-    // the oldest edit's sequence number.
-    TreeSet<Long> sequenceNumbers = new TreeSet<Long>(this.outputfiles.headMap(
-        oldestOutstandingSeqNum).keySet());
-    // Now remove old log files (if any)
-    if (LOG.isDebugEnabled()) {
-      if (sequenceNumbers.size() > 0) {
-        LOG.debug("Found " + sequenceNumbers.size() + " hlogs to remove" +
-          " out of total " + this.outputfiles.size() + ";" +
-          " oldest outstanding sequenceid is " + oldestOutstandingSeqNum);
-      }
-    }
-    for (Long seq : sequenceNumbers) {
-      archiveLogFile(this.outputfiles.remove(seq), seq);
+    for (Map.Entry<Path, Map<byte[], Long>> e : hlogSequenceNums.entrySet()) {
+      // iterate over the log file.
+      Path log = e.getKey();
+      Map<byte[], Long> sequenceNums = e.getValue();
+      // iterate over the map for this log file, and tell whether it should be archive or not.
+      if (areAllRegionsFlushed(sequenceNums, oldestFlushingSeqNumsLocal,
+        oldestUnflushedSeqNumsLocal)) {
+        logsToArchive.add(log);
+        LOG.debug("log file is ready for archiving " + log);
+      }
+    }
+    for (Path p : logsToArchive) {
+      archiveLogFile(p);
+      this.hlogSequenceNums.remove(p);
     }
   }
 
   /**
-   * Return regions that have edits that are equal or less than a certain sequence number.
-   * Static due to some old unit test.
-   * @param walSeqNum The sequence number to compare with.
-   * @param regionsToSeqNums Encoded region names to sequence ids
-   * @return All regions whose seqNum <= walSeqNum. Null if no regions found.
-   */
-  static byte[][] findMemstoresWithEditsEqualOrOlderThan(
-      final long walSeqNum, final Map<byte[], Long> regionsToSeqNums) {
-    List<byte[]> regions = null;
-    for (Map.Entry<byte[], Long> e : regionsToSeqNums.entrySet()) {
-      if (e.getValue().longValue() <= walSeqNum) {
-        if (regions == null) regions = new ArrayList<byte[]>();
-        regions.add(e.getKey());
+   * Takes a region:sequenceId map for a WAL file, and checks whether the file can be archived.
+   * It compares the region entries present in the passed sequenceNums map with the local copy of
+   * {@link #oldestUnflushedSeqNums} and {@link #oldestFlushingSeqNums}. If, for all regions,
+   * the value is lesser than the minimum of values present in the oldestFlushing/UnflushedSeqNums,
+   * then the wal file is eligible for archiving.
+   * @param sequenceNums for a HLog, at the time when it was rolled.
+   * @param oldestFlushingMap
+   * @param oldestUnflushedMap
+   * @return true if wal is eligible for archiving, false otherwise.
+   */
+   static boolean areAllRegionsFlushed(Map<byte[], Long> sequenceNums,
+      Map<byte[], Long> oldestFlushingMap, Map<byte[], Long> oldestUnflushedMap) {
+    for (Map.Entry<byte[], Long> regionSeqIdEntry : sequenceNums.entrySet()) {
+      // find region entries in the flushing/unflushed map. If there is no entry, it means
+      // a region doesn't have any unflushed entry.
+      long oldestFlushing = oldestFlushingMap.containsKey(regionSeqIdEntry.getKey()) ?
+          oldestFlushingMap.get(regionSeqIdEntry.getKey()) : Long.MAX_VALUE;
+      long oldestUnFlushed = oldestUnflushedMap.containsKey(regionSeqIdEntry.getKey()) ?
+          oldestUnflushedMap.get(regionSeqIdEntry.getKey()) : Long.MAX_VALUE;
+          // do a minimum to be sure to contain oldest sequence Id
+      long minSeqNum = Math.min(oldestFlushing, oldestUnFlushed);
+      if (minSeqNum <= regionSeqIdEntry.getValue()) return false;// can't archive
+    }
+    return true;
+  }
+
+  /**
+   * Iterates over the given map of regions, and compares their sequence numbers with corresponding
+   * entries in {@link #oldestUnflushedSeqNums}. If the sequence number is greater or equal, the
+   * region is eligible to flush, otherwise, there is no benefit to flush (from the perspective of
+   * passed regionsSequenceNums map), because the region has already flushed the entries present
+   * in the WAL file for which this method is called for (typically, the oldest wal file).
+   * @param regionsSequenceNums
+   * @return regions which should be flushed (whose sequence numbers are larger than their
+   * corresponding un-flushed entries.
+   */
+  private byte[][] findEligibleMemstoresToFlush(Map<byte[], Long> regionsSequenceNums) {
+    List<byte[]> regionsToFlush = null;
+    // Keeping the old behavior of iterating unflushedSeqNums under oldestSeqNumsLock.
+    synchronized (oldestSeqNumsLock) {
+      for (Map.Entry<byte[], Long> e : regionsSequenceNums.entrySet()) {
+        Long unFlushedVal = this.oldestUnflushedSeqNums.get(e.getKey());
+        if (unFlushedVal != null && unFlushedVal <= e.getValue()) {
+          if (regionsToFlush == null) regionsToFlush = new ArrayList<byte[]>();
+          regionsToFlush.add(e.getKey());
+        }
       }
     }
-    return regions == null ? null : regions
+    return regionsToFlush == null ? null : regionsToFlush
         .toArray(new byte[][] { HConstants.EMPTY_BYTE_ARRAY });
   }
 
-  private byte[][] getRegionsToForceFlush() throws IOException {
-    // If too many log files, figure which regions we need to flush.
-    // Array is an array of encoded region names.
+  /**
+   * If the number of un-archived WAL files is greater than maximum allowed, it checks
+   * the first (oldest) WAL file, and returns the regions which should be flushed so that it could
+   * be archived.
+   * @return regions to flush in order to archive oldest wal file.
+   * @throws IOException
+   */
+  byte[][] findRegionsToForceFlush() throws IOException {
     byte [][] regions = null;
     int logCount = getNumLogFiles();
     if (logCount > this.maxLogs && logCount > 0) {
-      // This is an array of encoded region names.
-      synchronized (oldestSeqNumsLock) {
-        regions = findMemstoresWithEditsEqualOrOlderThan(this.outputfiles.firstKey(),
-          this.oldestUnflushedSeqNums);
-      }
-      if (regions != null) {
-        StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < regions.length; i++) {
-          if (i > 0) sb.append(", ");
-          sb.append(Bytes.toStringBinary(regions[i]));
-        }
-        LOG.info("Too many hlogs: logs=" + logCount + ", maxlogs=" +
-           this.maxLogs + "; forcing flush of " + regions.length + " regions(s): " +
-           sb.toString());
-      }
+      Map.Entry<Path, Map<byte[], Long>> firstWALEntry =
+        this.hlogSequenceNums.firstEntry();
+      regions = findEligibleMemstoresToFlush(firstWALEntry.getValue());
+    }
+    if (regions != null) {
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < regions.length; i++) {
+        if (i > 0) sb.append(", ");
+        sb.append(Bytes.toStringBinary(regions[i]));
+      }
+      LOG.info("Too many hlogs: logs=" + logCount + ", maxlogs=" +
+         this.maxLogs + "; forcing flush of " + regions.length + " regions(s): " +
+         sb.toString());
     }
     return regions;
   }
 
   /*
-   * Cleans up current writer closing and adding to outputfiles.
+   * Cleans up current writer closing.
    * Presumes we're operating inside an updateLock scope.
    * @return Path to current writer or null if none.
    * @throws IOException
@@ -690,18 +744,13 @@ class FSHLog implements HLog, Syncable {
       }
       if (currentfilenum >= 0) {
         oldFile = computeFilename(currentfilenum);
-        this.outputfiles.put(Long.valueOf(this.logSeqNum.get()), oldFile);
       }
     }
     return oldFile;
   }
 
-  private void archiveLogFile(final Path p, final Long seqno) throws IOException {
+  private void archiveLogFile(final Path p) throws IOException {
     Path newPath = getHLogArchivePath(this.oldLogDir, p);
-    LOG.info("moving old hlog file " + FSUtils.getPath(p) +
-      " whose highest sequenceid is " + seqno + " to " +
-      FSUtils.getPath(newPath));
-
     // Tell our listeners that a log is going to be archived.
     if (!this.listeners.isEmpty()) {
       for (WALActionsListener i : this.listeners) {
@@ -745,6 +794,26 @@ class FSHLog implements HLog, Syncable {
     return new Path(dir, child);
   }
 
+/**
+ * A log file has a creation timestamp (in ms) in its file name ({@link #filenum}.
+ * This helper method returns the creation timestamp from a given log file.
+ * It extracts the timestamp assuming the filename is created with the
+ * {@link #computeFilename(long filenum)} method.
+ * @param fileName
+ * @return timestamp, as in the log file name.
+ */
+  protected long getFileNumFromFileName(Path fileName) {
+    if (fileName == null) throw new IllegalArgumentException("file name can't be null");
+    // The path should start with dir/<prefix>.
+    String prefixPathStr = new Path(dir, prefix + ".").toString();
+    if (!fileName.toString().startsWith(prefixPathStr)) {
+      throw new IllegalArgumentException("The log doesn't belong to this regionserver");
+    }
+    String chompedPath = fileName.toString().substring(prefixPathStr.length());
+    if (forMeta) chompedPath = chompedPath.substring(0, chompedPath.indexOf(META_HLOG_FILE_EXTN));
+    return Long.parseLong(chompedPath);
+  }
+
   @Override
   public void closeAndDelete() throws IOException {
     close();
@@ -835,15 +904,15 @@ class FSHLog implements HLog, Syncable {
 
   @Override
   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
-    final long now, HTableDescriptor htd)
+    final long now, HTableDescriptor htd, AtomicLong sequenceId)
   throws IOException {
-    append(info, tableName, edits, now, htd, true);
+    append(info, tableName, edits, now, htd, true, sequenceId);
   }
 
   @Override
-  public void append(HRegionInfo info, TableName tableName, WALEdit edits,
-    final long now, HTableDescriptor htd, boolean isInMemstore) throws IOException {
-    append(info, tableName, edits, new ArrayList<UUID>(), now, htd, true, isInMemstore);
+  public void append(HRegionInfo info, TableName tableName, WALEdit edits, final long now,
+      HTableDescriptor htd, boolean isInMemstore, AtomicLong sequenceId) throws IOException {
+    append(info, tableName, edits, new ArrayList<UUID>(), now, htd, true, isInMemstore, sequenceId);
   }
 
   /**
@@ -869,12 +938,13 @@ class FSHLog implements HLog, Syncable {
    * @param clusterIds that have consumed the change (for replication)
    * @param now
    * @param doSync shall we sync?
+   * @param sequenceId of the region.
    * @return txid of this transaction
    * @throws IOException
    */
   @SuppressWarnings("deprecation")
   private long append(HRegionInfo info, TableName tableName, WALEdit edits, List<UUID> clusterIds,
-      final long now, HTableDescriptor htd, boolean doSync, boolean isInMemstore)
+      final long now, HTableDescriptor htd, boolean doSync, boolean isInMemstore, AtomicLong sequenceId)
     throws IOException {
       if (edits.isEmpty()) return this.unflushedEntries.get();
       if (this.closed) {
@@ -884,7 +954,9 @@ class FSHLog implements HLog, Syncable {
       try {
         long txid = 0;
         synchronized (this.updateLock) {
-          long seqNum = obtainSeqNum();
+          // get the sequence number from the passed Long. In normal flow, it is coming from the
+          // region.
+          long seqNum = sequenceId.incrementAndGet();
           // The 'lastSeqWritten' map holds the sequence number of the oldest
           // write for each region (i.e. the first edit added to the particular
           // memstore). . When the cache is flushed, the entry for the
@@ -901,6 +973,7 @@ class FSHLog implements HLog, Syncable {
           if (htd.isDeferredLogFlush()) {
             lastDeferredTxid = txid;
           }
+          this.latestSequenceNums.put(encodedRegionName, seqNum);
         }
         // Sync if catalog region, and if not then check if that table supports
         // deferred log flushing
@@ -918,9 +991,9 @@ class FSHLog implements HLog, Syncable {
 
   @Override
   public long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
-      List<UUID> clusterIds, final long now, HTableDescriptor htd)
+      List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId)
     throws IOException {
-    return append(info, tableName, edits, clusterIds, now, htd, false, true);
+    return append(info, tableName, edits, clusterIds, now, htd, false, true, sequenceId);
   }
 
   /**
@@ -1246,21 +1319,18 @@ class FSHLog implements HLog, Syncable {
     return numEntries.get();
   }
 
-  @Override
-  public long obtainSeqNum() {
-    return this.logSeqNum.incrementAndGet();
-  }
-
   /** @return the number of log files in use */
   int getNumLogFiles() {
-    return outputfiles.size();
+    return hlogSequenceNums.size();
   }
 
   @Override
-  public Long startCacheFlush(final byte[] encodedRegionName) {
+  public boolean startCacheFlush(final byte[] encodedRegionName) {
     Long oldRegionSeqNum = null;
     if (!closeBarrier.beginOp()) {
-      return null;
+      LOG.info("Flush will not be started for " + Bytes.toString(encodedRegionName) +
+        " - because the server is closing.");
+      return false;
     }
     synchronized (oldestSeqNumsLock) {
       oldRegionSeqNum = this.oldestUnflushedSeqNums.remove(encodedRegionName);
@@ -1279,7 +1349,7 @@ class FSHLog implements HLog, Syncable {
       LOG.warn("Couldn't find oldest seqNum for the region we are about to flush: ["
         + Bytes.toString(encodedRegionName) + "]");
     }
-    return obtainSeqNum();
+    return true;
   }
 
   @Override

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Thu Nov  7 18:10:38 2013
@@ -24,6 +24,7 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
 
 import org.apache.commons.logging.Log;
@@ -205,19 +206,6 @@ public interface HLog {
   // TODO: Remove.  Implementation detail.
   long getFilenum();
 
-  /**
-   * Called to ensure that log sequence numbers are always greater
-   *
-   * @param newvalue We'll set log edit/sequence number to this value if it is greater
-   * than the current value.
-   */
-  void setSequenceNumber(final long newvalue);
-
-  /**
-   * @return log sequence number
-   */
-  long getSequenceNumber();
-
   // TODO: Log rolling should not be in this interface.
   /**
    * Roll the log writer. That is, start writing log messages to a new file.
@@ -270,9 +258,10 @@ public interface HLog {
   /**
    * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor),
    * except it causes a sync on the log
+   * @param sequenceId of the region.
    */
   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
-      final long now, HTableDescriptor htd) throws IOException;
+      final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException;
 
   /**
    * Append a set of edits to the log. Log edits are keyed by (encoded)
@@ -284,9 +273,10 @@ public interface HLog {
    * @param now
    * @param htd
    * @param isInMemstore Whether the record is in memstore. False for system records.
+   * @param sequenceId of the region.
    */
-  public void append(HRegionInfo info, TableName tableName, WALEdit edits,
-      final long now, HTableDescriptor htd, boolean isInMemstore) throws IOException;
+  public void append(HRegionInfo info, TableName tableName, WALEdit edits, final long now,
+      HTableDescriptor htd, boolean isInMemstore, AtomicLong sequenceId) throws IOException;
 
   /**
    * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and
@@ -297,11 +287,12 @@ public interface HLog {
    * @param clusterIds The clusters that have consumed the change (for replication)
    * @param now
    * @param htd
+   * @param sequenceId of the region
    * @return txid of this transaction
    * @throws IOException
    */
   public long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
-      List<UUID> clusterIds, final long now, HTableDescriptor htd) throws IOException;
+      List<UUID> clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId) throws IOException;
 
   // TODO: Do we need all these versions of sync?
   void hsync() throws IOException;
@@ -313,27 +304,20 @@ public interface HLog {
   void sync(long txid) throws IOException;
 
   /**
-   * Obtain a log sequence number.
-   */
-  // TODO: Name better to differentiate from getSequenceNumber.
-  long obtainSeqNum();
-
-  /**
    * WAL keeps track of the sequence numbers that were not yet flushed from memstores
    * in order to be able to do cleanup. This method tells WAL that some region is about
    * to flush memstore.
    *
    * We stash the oldest seqNum for the region, and let the the next edit inserted in this
-   * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor)}
-   * as new oldest seqnum. In case of flush being aborted, we put the stashed value back;
-   * in case of flush succeeding, the seqNum of that first edit after start becomes the
-   * valid oldest seqNum for this region.
-   *
-   * @return current seqNum, to pass on to flushers (who will put it into the metadata of
-   *         the resulting file as an upper-bound seqNum for that file), or NULL if flush
-   *         should not be started.
+   * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor,
+   * AtomicLong)} as new oldest seqnum.
+   * In case of flush being aborted, we put the stashed value back; in case of flush succeeding,
+   * the seqNum of that first edit after start becomes the valid oldest seqNum for this region.
+   *
+   * @return true if the flush can proceed, false in case wal is closing (ususally, when server is
+   * closing) and flush couldn't be started.
    */
-  Long startCacheFlush(final byte[] encodedRegionName);
+  boolean startCacheFlush(final byte[] encodedRegionName);
 
   /**
    * Complete the cache flush.

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogUtil.java Thu Nov  7 18:10:38 2013
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.util.NavigableSet;
 import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -256,12 +257,13 @@ public class HLogUtil {
    * This provides info to the HMaster to allow it to recover the compaction if
    * this regionserver dies in the middle (This part is not yet implemented). It also prevents
    * the compaction from finishing if this regionserver has already lost its lease on the log.
+   * @param sequenceId Used by HLog to get sequence Id for the waledit.
    */
   public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
-      final CompactionDescriptor c) throws IOException {
+      final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
     WALEdit e = WALEdit.createCompaction(c);
     log.append(info, TableName.valueOf(c.getTableName().toByteArray()), e,
-        EnvironmentEdgeManager.currentTimeMillis(), htd, false);
+        EnvironmentEdgeManager.currentTimeMillis(), htd, false, sequenceId);
     if (LOG.isTraceEnabled()) {
       LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
     }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java Thu Nov  7 18:10:38 2013
@@ -49,6 +49,7 @@ import java.security.PrivilegedException
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.junit.Assert.*;
 
@@ -138,6 +139,7 @@ public class TestWALObserver {
     Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE));
     deleteDir(basedir);
     fs.mkdirs(new Path(basedir, hri.getEncodedName()));
+    final AtomicLong sequenceId = new AtomicLong(0);
 
     HLog log = HLogFactory.createHLog(this.fs, hbaseRootDir,
         TestWALObserver.class.getName(), this.conf);
@@ -186,7 +188,7 @@ public class TestWALObserver {
 
     // it's where WAL write cp should occur.
     long now = EnvironmentEdgeManager.currentTimeMillis();
-    log.append(hri, hri.getTable(), edit, now, htd);
+    log.append(hri, hri.getTable(), edit, now, htd, sequenceId);
 
     // the edit shall have been change now by the coprocessor.
     foundFamily0 = false;
@@ -222,6 +224,7 @@ public class TestWALObserver {
     // ultimately called by HRegion::initialize()
     TableName tableName = TableName.valueOf("testWALCoprocessorReplay");
     final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
+    final AtomicLong sequenceId = new AtomicLong(0);
     // final HRegionInfo hri =
     // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
     // final HRegionInfo hri1 =
@@ -247,9 +250,9 @@ public class TestWALObserver {
       // addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
       // EnvironmentEdgeManager.getDelegate(), wal);
       addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
-          EnvironmentEdgeManager.getDelegate(), wal, htd);
+          EnvironmentEdgeManager.getDelegate(), wal, htd, sequenceId);
     }
-    wal.append(hri, tableName, edit, now, htd);
+    wal.append(hri, tableName, edit, now, htd, sequenceId);
     // sync to fs.
     wal.sync();
 
@@ -369,7 +372,7 @@ public class TestWALObserver {
 
   private void addWALEdits(final TableName tableName, final HRegionInfo hri,
       final byte[] rowName, final byte[] family, final int count,
-      EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
+      EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd, final AtomicLong sequenceId)
       throws IOException {
     String familyStr = Bytes.toString(family);
     for (int j = 0; j < count; j++) {
@@ -378,7 +381,7 @@ public class TestWALObserver {
       WALEdit edit = new WALEdit();
       edit.add(new KeyValue(rowName, family, qualifierBytes, ee
           .currentTimeMillis(), columnBytes));
-      wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
+      wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
     }
   }
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHLogRecordReader.java Thu Nov  7 18:10:38 2013
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -114,11 +115,12 @@ public class TestHLogRecordReader {
     HLog log = HLogFactory.createHLog(fs, hbaseDir, logName, conf);
     long ts = System.currentTimeMillis();
     WALEdit edit = new WALEdit();
+    final AtomicLong sequenceId = new AtomicLong(0);
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"), ts, value));
-    log.append(info, tableName, edit, ts, htd);
+    log.append(info, tableName, edit, ts, htd, sequenceId);
     edit = new WALEdit();
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"), ts+1, value));
-    log.append(info, tableName, edit, ts+1, htd);
+    log.append(info, tableName, edit, ts+1, htd, sequenceId);
     log.rollWriter();
 
     Thread.sleep(1);
@@ -126,10 +128,10 @@ public class TestHLogRecordReader {
 
     edit = new WALEdit();
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("3"), ts1+1, value));
-    log.append(info, tableName, edit, ts1+1, htd);
+    log.append(info, tableName, edit, ts1+1, htd, sequenceId);
     edit = new WALEdit();
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("4"), ts1+2, value));
-    log.append(info, tableName, edit, ts1+2, htd);
+    log.append(info, tableName, edit, ts1+2, htd, sequenceId);
     log.close();
 
     HLogInputFormat input = new HLogInputFormat();
@@ -161,11 +163,12 @@ public class TestHLogRecordReader {
   public void testHLogRecordReader() throws Exception {
     HLog log = HLogFactory.createHLog(fs, hbaseDir, logName, conf);
     byte [] value = Bytes.toBytes("value");
+    final AtomicLong sequenceId = new AtomicLong(0);
     WALEdit edit = new WALEdit();
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("1"),
         System.currentTimeMillis(), value));
     log.append(info, tableName, edit,
-      System.currentTimeMillis(), htd);
+      System.currentTimeMillis(), htd, sequenceId);
 
     Thread.sleep(1); // make sure 2nd log gets a later timestamp
     long secondTs = System.currentTimeMillis();
@@ -175,7 +178,7 @@ public class TestHLogRecordReader {
     edit.add(new KeyValue(rowName, family, Bytes.toBytes("2"),
         System.currentTimeMillis(), value));
     log.append(info, tableName, edit,
-      System.currentTimeMillis(), htd);
+      System.currentTimeMillis(), htd, sequenceId);
     log.close();
     long thirdTs = System.currentTimeMillis();
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java Thu Nov  7 18:10:38 2013
@@ -252,7 +252,6 @@ public class TestDistributedLogSplitting
   @Test(timeout = 300000)
   public void testLogReplayWithNonMetaRSDown() throws Exception {
     LOG.info("testLogReplayWithNonMetaRSDown");
-    conf.setLong("hbase.regionserver.hlog.blocksize", 100*1024);
     conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
     startCluster(NUM_RS);
     final int NUM_REGIONS_TO_CREATE = 40;
@@ -1144,6 +1143,9 @@ public class TestDistributedLogSplitting
     TableName fullTName = TableName.valueOf(tname);
     // remove root and meta region
     regions.remove(HRegionInfo.FIRST_META_REGIONINFO);
+    // using one sequenceId for edits across all regions is ok.
+    final AtomicLong sequenceId = new AtomicLong(10);
+
 
     for(Iterator<HRegionInfo> iter = regions.iterator(); iter.hasNext(); ) {
       HRegionInfo regionInfo = iter.next();
@@ -1183,7 +1185,7 @@ public class TestDistributedLogSplitting
                                              // key
         byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
         e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
-        log.append(curRegionInfo, fullTName, e, System.currentTimeMillis(), htd);
+        log.append(curRegionInfo, fullTName, e, System.currentTimeMillis(), htd, sequenceId);
         counts[i % n] += 1;
       }
     }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Thu Nov  7 18:10:38 2013
@@ -53,6 +53,7 @@ import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.Log;
@@ -462,7 +463,7 @@ public class TestHRegion {
           .getRegionFileSystem().getStoreDir(Bytes.toString(family)));
 
       HLogUtil.writeCompactionMarker(region.getLog(), this.region.getTableDesc(),
-          this.region.getRegionInfo(), compactionDescriptor);
+          this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));
 
       Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
 
@@ -3799,10 +3800,10 @@ public class TestHRegion {
     put.add(family, Bytes.toBytes("q1"), Bytes.toBytes("v1"));
     put.setDurability(mutationDurability);
     region.put(put);
-
-    // verify append called or not
-    verify(log, expectAppend ? times(1) : never()).appendNoSync((HRegionInfo) any(), eq(tableName),
-        (WALEdit) any(), (List<UUID>) any(), anyLong(), (HTableDescriptor) any());
+    //verify append called or not
+    verify(log, expectAppend ? times(1) : never())
+      .appendNoSync((HRegionInfo)any(), eq(tableName), (WALEdit)any(), (List<UUID>)any(),
+        anyLong(), (HTableDescriptor)any(), (AtomicLong)any());
 
     // verify sync called or not
     if (expectSync || expectSyncFromLogSyncer) {

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java Thu Nov  7 18:10:38 2013
@@ -20,10 +20,12 @@ package org.apache.hadoop.hbase.regionse
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -106,9 +108,10 @@ public final class HLogPerformanceEvalua
           addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
           HRegionInfo hri = region.getRegionInfo();
           if (this.noSync) {
-            hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd);
+            hlog.appendNoSync(hri, hri.getTable(), walEdit, new ArrayList<UUID>(), now, htd,
+              region.getSequenceId());
           } else {
-            hlog.append(hri, hri.getTable(), walEdit, now, htd);
+            hlog.append(hri, hri.getTable(), walEdit, now, htd, region.getSequenceId());
           }
         }
         long totalTime = (System.currentTimeMillis() - startTime);
@@ -251,16 +254,15 @@ public final class HLogPerformanceEvalua
 
   /**
    * Verify the content of the WAL file.
-   * Verify that sequenceids are ascending and that the file has expected number
-   * of edits.
+   * Verify that the file has expected number of edits.
    * @param wal
    * @return Count of edits.
    * @throws IOException
    */
   private long verify(final Path wal, final boolean verbose) throws IOException {
     HLog.Reader reader = HLogFactory.createReader(wal.getFileSystem(getConf()), wal, getConf());
-    long previousSeqid = -1;
     long count = 0;
+    Map<String, Long> sequenceIds = new HashMap<String, Long>();
     try {
       while (true) {
         Entry e = reader.next();
@@ -270,12 +272,17 @@ public final class HLogPerformanceEvalua
         }
         count++;
         long seqid = e.getKey().getLogSeqNum();
-        if (verbose) LOG.info("seqid=" + seqid);
-        if (previousSeqid >= seqid) {
-          throw new IllegalStateException("wal=" + wal.getName() +
-            ", previousSeqid=" + previousSeqid + ", seqid=" + seqid);
+        if (sequenceIds.containsKey(Bytes.toString(e.getKey().getEncodedRegionName()))) {
+          // sequenceIds should be increasing for every regions
+          if (sequenceIds.get(Bytes.toString(e.getKey().getEncodedRegionName())) >= seqid) {
+            throw new IllegalStateException("wal = " + wal.getName() + ", " + "previous seqid = "
+                + sequenceIds.get(Bytes.toString(e.getKey().getEncodedRegionName()))
+                + ", current seqid = " + seqid);
+          }
+        } else {
+          sequenceIds.put(Bytes.toString(e.getKey().getEncodedRegionName()), seqid);
         }
-        previousSeqid = seqid;
+        if (verbose) LOG.info("seqid=" + seqid);
       }
     } finally {
       reader.close();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java Thu Nov  7 18:10:38 2013
@@ -23,9 +23,14 @@ import static org.junit.Assert.*;
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.net.BindException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.TreeMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -58,6 +63,9 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 /** JUnit test case for HLog */
 @Category(LargeTests.class)
@@ -136,15 +144,13 @@ public class TestHLog  {
   }
 
   /**
-   * Test that with three concurrent threads we still write edits in sequence
-   * edit id order.
+   * Write to a log file with three concurrent threads and verifying all data is written.
    * @throws Exception
    */
   @Test
-  public void testMaintainOrderWithConcurrentWrites() throws Exception {
+  public void testConcurrentWrites() throws Exception {
     // Run the HPE tool with three threads writing 3000 edits each concurrently.
-    // When done, verify that all edits were written and that the order in the
-    // WALs is of ascending edit sequence ids.
+    // When done, verify that all edits were written.
     int errCode = HLogPerformanceEvaluation.
       innerMain(new Configuration(TEST_UTIL.getConfiguration()),
         new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
@@ -179,6 +185,7 @@ public class TestHLog  {
     htd.addFamily(new HColumnDescriptor("column"));
 
     // Add edits for three regions.
+    final AtomicLong sequenceId = new AtomicLong(1);
     try {
       for (int ii = 0; ii < howmany; ii++) {
         for (int i = 0; i < howmany; i++) {
@@ -192,7 +199,7 @@ public class TestHLog  {
                 System.currentTimeMillis(), column));
             LOG.info("Region " + i + ": " + edit);
             log.append(infos[i], tableName, edit,
-              System.currentTimeMillis(), htd);
+              System.currentTimeMillis(), htd, sequenceId);
           }
         }
         log.rollWriter();
@@ -242,7 +249,7 @@ public class TestHLog  {
     in.close();
 
     HLog wal = HLogFactory.createHLog(fs, dir, "hlogdir", conf);
-
+    final AtomicLong sequenceId = new AtomicLong(1);
     final int total = 20;
     HLog.Reader reader = null;
 
@@ -255,7 +262,7 @@ public class TestHLog  {
       for (int i = 0; i < total; i++) {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
-        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd);
+        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
       }
       // Now call sync and try reading.  Opening a Reader before you sync just
       // gives you EOFE.
@@ -273,7 +280,7 @@ public class TestHLog  {
       for (int i = 0; i < total; i++) {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
-        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd);
+        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
       }
       reader = HLogFactory.createReader(fs, walPath, conf);
       count = 0;
@@ -292,7 +299,7 @@ public class TestHLog  {
       for (int i = 0; i < total; i++) {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));
-        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd);
+        wal.append(info, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
       }
       // Now I should have written out lots of blocks.  Sync then read.
       wal.sync();
@@ -314,34 +321,6 @@ public class TestHLog  {
     }
   }
 
-  /**
-   * Test the findMemstoresWithEditsEqualOrOlderThan method.
-   * @throws IOException
-   */
-  @Test
-  public void testFindMemstoresWithEditsEqualOrOlderThan() throws IOException {
-    Map<byte [], Long> regionsToSeqids = new TreeMap<byte [], Long>(Bytes.BYTES_COMPARATOR);
-    for (int i = 0; i < 10; i++) {
-      Long l = Long.valueOf(i);
-      regionsToSeqids.put(l.toString().getBytes(), l);
-    }
-    byte [][] regions =
-      FSHLog.findMemstoresWithEditsEqualOrOlderThan(1, regionsToSeqids);
-    assertEquals(2, regions.length);
-    assertTrue(Bytes.equals(regions[0], "0".getBytes()) ||
-        Bytes.equals(regions[0], "1".getBytes()));
-    regions = FSHLog.findMemstoresWithEditsEqualOrOlderThan(3, regionsToSeqids);
-    int count = 4;
-    assertEquals(count, regions.length);
-    // Regions returned are not ordered.
-    for (int i = 0; i < count; i++) {
-      assertTrue(Bytes.equals(regions[i], "0".getBytes()) ||
-        Bytes.equals(regions[i], "1".getBytes()) ||
-        Bytes.equals(regions[i], "2".getBytes()) ||
-        Bytes.equals(regions[i], "3".getBytes()));
-    }
-  }
-
   private void verifySplits(List<Path> splits, final int howmany)
   throws IOException {
     assertEquals(howmany * howmany, splits.size());
@@ -391,6 +370,7 @@ public class TestHLog  {
 
     HLog wal = HLogFactory.createHLog(fs, dir, "hlogdir",
         "hlogdir_archive", conf);
+    final AtomicLong sequenceId = new AtomicLong(1);
     final int total = 20;
 
     HTableDescriptor htd = new HTableDescriptor();
@@ -399,7 +379,7 @@ public class TestHLog  {
     for (int i = 0; i < total; i++) {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
-      wal.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd);
+      wal.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
     }
     // Now call sync to send the data to HDFS datanodes
     wal.sync();
@@ -513,6 +493,7 @@ public class TestHLog  {
     HLog log = null;
     try {
       log = HLogFactory.createHLog(fs, hbaseDir, getName(), conf);
+      final AtomicLong sequenceId = new AtomicLong(1);
 
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -528,7 +509,7 @@ public class TestHLog  {
       HTableDescriptor htd = new HTableDescriptor();
       htd.addFamily(new HColumnDescriptor("column"));
 
-      log.append(info, tableName, cols, System.currentTimeMillis(), htd);
+      log.append(info, tableName, cols, System.currentTimeMillis(), htd, sequenceId);
       log.startCacheFlush(info.getEncodedNameAsBytes());
       log.completeCacheFlush(info.getEncodedNameAsBytes());
       log.close();
@@ -571,6 +552,7 @@ public class TestHLog  {
     final byte [] row = Bytes.toBytes("row");
     Reader reader = null;
     HLog log = HLogFactory.createHLog(fs, hbaseDir, getName(), conf);
+    final AtomicLong sequenceId = new AtomicLong(1);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -585,7 +567,7 @@ public class TestHLog  {
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
       HTableDescriptor htd = new HTableDescriptor();
       htd.addFamily(new HColumnDescriptor("column"));
-      log.append(hri, tableName, cols, System.currentTimeMillis(), htd);
+      log.append(hri, tableName, cols, System.currentTimeMillis(), htd, sequenceId);
       log.startCacheFlush(hri.getEncodedNameAsBytes());
       log.completeCacheFlush(hri.getEncodedNameAsBytes());
       log.close();
@@ -626,6 +608,7 @@ public class TestHLog  {
         TableName.valueOf("tablename");
     final byte [] row = Bytes.toBytes("row");
     HLog log = HLogFactory.createHLog(fs, hbaseDir, getName(), conf);
+    final AtomicLong sequenceId = new AtomicLong(1);
     try {
       DumbWALActionsListener visitor = new DumbWALActionsListener();
       log.registerWALActionsListener(visitor);
@@ -640,7 +623,7 @@ public class TestHLog  {
         cols.add(new KeyValue(row, Bytes.toBytes("column"),
             Bytes.toBytes(Integer.toString(i)),
             timestamp, new byte[]{(byte) (i + '0')}));
-        log.append(hri, tableName, cols, System.currentTimeMillis(), htd);
+        log.append(hri, tableName, cols, System.currentTimeMillis(), htd, sequenceId);
       }
       assertEquals(COL_COUNT, visitor.increments);
       log.unregisterWALActionsListener(visitor);
@@ -648,7 +631,7 @@ public class TestHLog  {
       cols.add(new KeyValue(row, Bytes.toBytes("column"),
           Bytes.toBytes(Integer.toString(11)),
           timestamp, new byte[]{(byte) (11 + '0')}));
-      log.append(hri, tableName, cols, System.currentTimeMillis(), htd);
+      log.append(hri, tableName, cols, System.currentTimeMillis(), htd, sequenceId);
       assertEquals(COL_COUNT, visitor.increments);
     } finally {
       if (log != null) log.closeAndDelete();
@@ -665,6 +648,7 @@ public class TestHLog  {
 
     HLog log = HLogFactory.createHLog(fs, hbaseDir,
         getName(), conf);
+    final AtomicLong sequenceId = new AtomicLong(1);
     try {
       HRegionInfo hri = new HRegionInfo(tableName,
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
@@ -673,26 +657,26 @@ public class TestHLog  {
 
       // Add a single edit and make sure that rolling won't remove the file
       // Before HBASE-3198 it used to delete it
-      addEdits(log, hri, tableName, 1);
+      addEdits(log, hri, tableName, 1, sequenceId);
       log.rollWriter();
       assertEquals(1, ((FSHLog) log).getNumLogFiles());
 
       // See if there's anything wrong with more than 1 edit
-      addEdits(log, hri, tableName, 2);
+      addEdits(log, hri, tableName, 2, sequenceId);
       log.rollWriter();
       assertEquals(2, ((FSHLog) log).getNumLogFiles());
 
       // Now mix edits from 2 regions, still no flushing
-      addEdits(log, hri, tableName, 1);
-      addEdits(log, hri2, tableName2, 1);
-      addEdits(log, hri, tableName, 1);
-      addEdits(log, hri2, tableName2, 1);
+      addEdits(log, hri, tableName, 1, sequenceId);
+      addEdits(log, hri2, tableName2, 1, sequenceId);
+      addEdits(log, hri, tableName, 1, sequenceId);
+      addEdits(log, hri2, tableName2, 1, sequenceId);
       log.rollWriter();
       assertEquals(3, ((FSHLog) log).getNumLogFiles());
 
       // Flush the first region, we expect to see the first two files getting
       // archived. We need to append something or writer won't be rolled.
-      addEdits(log, hri2, tableName2, 1);
+      addEdits(log, hri2, tableName2, 1, sequenceId);
       log.startCacheFlush(hri.getEncodedNameAsBytes());
       log.completeCacheFlush(hri.getEncodedNameAsBytes());
       log.rollWriter();
@@ -701,7 +685,7 @@ public class TestHLog  {
       // Flush the second region, which removes all the remaining output files
       // since the oldest was completely flushed and the two others only contain
       // flush information
-      addEdits(log, hri2, tableName2, 1);
+      addEdits(log, hri2, tableName2, 1, sequenceId);
       log.startCacheFlush(hri2.getEncodedNameAsBytes());
       log.completeCacheFlush(hri2.getEncodedNameAsBytes());
       log.rollWriter();
@@ -778,7 +762,7 @@ public class TestHLog  {
   }
 
   private void addEdits(HLog log, HRegionInfo hri, TableName tableName,
-                        int times) throws IOException {
+                        int times, AtomicLong sequenceId) throws IOException {
     HTableDescriptor htd = new HTableDescriptor();
     htd.addFamily(new HColumnDescriptor("row"));
 
@@ -787,7 +771,7 @@ public class TestHLog  {
       long timestamp = System.currentTimeMillis();
       WALEdit cols = new WALEdit();
       cols.add(new KeyValue(row, row, row, timestamp, row));
-      log.append(hri, tableName, cols, timestamp, htd);
+      log.append(hri, tableName, cols, timestamp, htd, sequenceId);
     }
   }
 
@@ -949,6 +933,267 @@ public class TestHLog  {
     }
   }
 
+  /**
+   * tests the log comparator. Ensure that we are not mixing meta logs with non-meta logs (throws
+   * exception if we do). Comparison is based on the timestamp present in the wal name.
+   * @throws Exception
+   */
+  @Test
+  public void testHLogComparator() throws Exception {
+    HLog hlog1 = null;
+    HLog hlogMeta = null;
+    try {
+      hlog1 = HLogFactory.createHLog(fs, FSUtils.getRootDir(conf), dir.toString(), conf);
+      LOG.debug("Log obtained is: " + hlog1);
+      Comparator<Path> comp = ((FSHLog) hlog1).LOG_NAME_COMPARATOR;
+      Path p1 = ((FSHLog) hlog1).computeFilename(11);
+      Path p2 = ((FSHLog) hlog1).computeFilename(12);
+      // comparing with itself returns 0
+      assertTrue(comp.compare(p1, p1) == 0);
+      // comparing with different filenum.
+      assertTrue(comp.compare(p1, p2) < 0);
+      hlogMeta = HLogFactory.createMetaHLog(fs, FSUtils.getRootDir(conf), dir.toString(), conf,
+        null, null);
+      Comparator<Path> compMeta = ((FSHLog) hlogMeta).LOG_NAME_COMPARATOR;
+
+      Path p1WithMeta = ((FSHLog) hlogMeta).computeFilename(11);
+      Path p2WithMeta = ((FSHLog) hlogMeta).computeFilename(12);
+      assertTrue(compMeta.compare(p1WithMeta, p1WithMeta) == 0);
+      assertTrue(compMeta.compare(p1WithMeta, p2WithMeta) < 0);
+      // mixing meta and non-meta logs gives error
+      boolean ex = false;
+      try {
+        comp.compare(p1WithMeta, p2);
+      } catch (Exception e) {
+        ex = true;
+      }
+      assertTrue("Comparator doesn't complain while checking meta log files", ex);
+      boolean exMeta = false;
+      try {
+        compMeta.compare(p1WithMeta, p2);
+      } catch (Exception e) {
+        exMeta = true;
+      }
+      assertTrue("Meta comparator doesn't complain while checking log files", exMeta);
+    } finally {
+      if (hlog1 != null) hlog1.close();
+      if (hlogMeta != null) hlogMeta.close();
+    }
+  }
+
+  /**
+   * Tests wal archiving by adding data, doing flushing/rolling and checking we archive old logs
+   * and also don't archive "live logs" (that is, a log with un-flushed entries).
+   * <p>
+   * This is what it does:
+   * It creates two regions, and does a series of inserts along with log rolling.
+   * Whenever a WAL is rolled, FSHLog checks previous wals for archiving. A wal is eligible for
+   * archiving if for all the regions which have entries in that wal file, have flushed - past
+   * their maximum sequence id in that wal file.
+   * <p>
+   * @throws IOException
+   */
+  @Test
+  public void testWALArchiving() throws IOException {
+    LOG.debug("testWALArchiving");
+    TableName table1 = TableName.valueOf("t1");
+    TableName table2 = TableName.valueOf("t2");
+    HLog hlog = HLogFactory.createHLog(fs, FSUtils.getRootDir(conf), dir.toString(), conf);
+    try {
+      assertEquals(0, ((FSHLog) hlog).getNumLogFiles());
+      HRegionInfo hri1 = new HRegionInfo(table1, HConstants.EMPTY_START_ROW,
+          HConstants.EMPTY_END_ROW);
+      HRegionInfo hri2 = new HRegionInfo(table2, HConstants.EMPTY_START_ROW,
+          HConstants.EMPTY_END_ROW);
+      // ensure that we don't split the regions.
+      hri1.setSplit(false);
+      hri2.setSplit(false);
+      // variables to mock region sequenceIds.
+      final AtomicLong sequenceId1 = new AtomicLong(1);
+      final AtomicLong sequenceId2 = new AtomicLong(1);
+      // start with the testing logic: insert a waledit, and roll writer
+      addEdits(hlog, hri1, table1, 1, sequenceId1);
+      hlog.rollWriter();
+      // assert that the wal is rolled
+      assertEquals(1, ((FSHLog) hlog).getNumLogFiles());
+      // add edits in the second wal file, and roll writer.
+      addEdits(hlog, hri1, table1, 1, sequenceId1);
+      hlog.rollWriter();
+      // assert that the wal is rolled
+      assertEquals(2, ((FSHLog) hlog).getNumLogFiles());
+      // add a waledit to table1, and flush the region.
+      addEdits(hlog, hri1, table1, 3, sequenceId1);
+      flushRegion(hlog, hri1.getEncodedNameAsBytes());
+      // roll log; all old logs should be archived.
+      hlog.rollWriter();
+      assertEquals(0, ((FSHLog) hlog).getNumLogFiles());
+      // add an edit to table2, and roll writer
+      addEdits(hlog, hri2, table2, 1, sequenceId2);
+      hlog.rollWriter();
+      assertEquals(1, ((FSHLog) hlog).getNumLogFiles());
+      // add edits for table1, and roll writer
+      addEdits(hlog, hri1, table1, 2, sequenceId1);
+      hlog.rollWriter();
+      assertEquals(2, ((FSHLog) hlog).getNumLogFiles());
+      // add edits for table2, and flush hri1.
+      addEdits(hlog, hri2, table2, 2, sequenceId2);
+      flushRegion(hlog, hri1.getEncodedNameAsBytes());
+      // the log : region-sequenceId map is
+      // log1: region2 (unflushed)
+      // log2: region1 (flushed)
+      // log3: region2 (unflushed)
+      // roll the writer; log2 should be archived.
+      hlog.rollWriter();
+      assertEquals(2, ((FSHLog) hlog).getNumLogFiles());
+      // flush region2, and all logs should be archived.
+      addEdits(hlog, hri2, table2, 2, sequenceId2);
+      flushRegion(hlog, hri2.getEncodedNameAsBytes());
+      hlog.rollWriter();
+      assertEquals(0, ((FSHLog) hlog).getNumLogFiles());
+    } finally {
+      if (hlog != null) hlog.close();
+    }
+  }
+
+  /**
+   * On rolling a wal after reaching the threshold, {@link HLog#rollWriter()} returns the list of
+   * regions which should be flushed in order to archive the oldest wal file.
+   * <p>
+   * This method tests this behavior by inserting edits and rolling the wal enough times to reach
+   * the max number of logs threshold. It checks whether we get the "right regions" for flush on
+   * rolling the wal.
+   * @throws Exception
+   */
+  @Test
+  public void testFindMemStoresEligibleForFlush() throws Exception {
+    LOG.debug("testFindMemStoresEligibleForFlush");
+    Configuration conf1 = HBaseConfiguration.create(conf);
+    conf1.setInt("hbase.regionserver.maxlogs", 1);
+    HLog hlog = HLogFactory.createHLog(fs, FSUtils.getRootDir(conf1), dir.toString(), conf1);
+    TableName t1 = TableName.valueOf("t1");
+    TableName t2 = TableName.valueOf("t2");
+    HRegionInfo hri1 = new HRegionInfo(t1, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+    HRegionInfo hri2 = new HRegionInfo(t2, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+    // variables to mock region sequenceIds
+    final AtomicLong sequenceId1 = new AtomicLong(1);
+    final AtomicLong sequenceId2 = new AtomicLong(1);
+    // add edits and roll the wal
+    try {
+      addEdits(hlog, hri1, t1, 2, sequenceId1);
+      hlog.rollWriter();
+      // add some more edits and roll the wal. This would reach the log number threshold
+      addEdits(hlog, hri1, t1, 2, sequenceId1);
+      hlog.rollWriter();
+      // with above rollWriter call, the max logs limit is reached.
+      assertTrue(((FSHLog) hlog).getNumLogFiles() == 2);
+
+      // get the regions to flush; since there is only one region in the oldest wal, it should
+      // return only one region.
+      byte[][] regionsToFlush = ((FSHLog) hlog).findRegionsToForceFlush();
+      assertEquals(regionsToFlush.length, 1);
+      assertEquals(regionsToFlush[0], hri1.getEncodedNameAsBytes());
+      // insert edits in second region
+      addEdits(hlog, hri2, t2, 2, sequenceId2);
+      // get the regions to flush, it should still read region1.
+      regionsToFlush = ((FSHLog) hlog).findRegionsToForceFlush();
+      assertEquals(regionsToFlush.length, 1);
+      assertEquals(regionsToFlush[0], hri1.getEncodedNameAsBytes());
+      // flush region 1, and roll the wal file. Only last wal which has entries for region1 should
+      // remain.
+      flushRegion(hlog, hri1.getEncodedNameAsBytes());
+      hlog.rollWriter();
+      // only one wal should remain now (that is for the second region).
+      assertEquals(1, ((FSHLog) hlog).getNumLogFiles());
+      // flush the second region
+      flushRegion(hlog, hri2.getEncodedNameAsBytes());
+      hlog.rollWriter(true);
+      // no wal should remain now.
+      assertEquals(0, ((FSHLog) hlog).getNumLogFiles());
+      // add edits both to region 1 and region 2, and roll.
+      addEdits(hlog, hri1, t1, 2, sequenceId1);
+      addEdits(hlog, hri2, t2, 2, sequenceId2);
+      hlog.rollWriter();
+      // add edits and roll the writer, to reach the max logs limit.
+      assertEquals(1, ((FSHLog) hlog).getNumLogFiles());
+      addEdits(hlog, hri1, t1, 2, sequenceId1);
+      hlog.rollWriter();
+      // it should return two regions to flush, as the oldest wal file has entries
+      // for both regions.
+      regionsToFlush = ((FSHLog) hlog).findRegionsToForceFlush();
+      assertEquals(regionsToFlush.length, 2);
+      // flush both regions
+      flushRegion(hlog, hri1.getEncodedNameAsBytes());
+      flushRegion(hlog, hri2.getEncodedNameAsBytes());
+      hlog.rollWriter(true);
+      assertEquals(0, ((FSHLog) hlog).getNumLogFiles());
+      // Add an edit to region1, and roll the wal.
+      addEdits(hlog, hri1, t1, 2, sequenceId1);
+      // tests partial flush: roll on a partial flush, and ensure that wal is not archived.
+      hlog.startCacheFlush(hri1.getEncodedNameAsBytes());
+      hlog.rollWriter();
+      hlog.completeCacheFlush(hri1.getEncodedNameAsBytes());
+      assertEquals(1, ((FSHLog) hlog).getNumLogFiles());
+    } finally {
+      if (hlog != null) hlog.close();
+    }
+  }
+
+  /**
+   * Simulates HLog append ops for a region and tests
+   * {@link FSHLog#areAllRegionsFlushed(Map, Map, Map)} API.
+   * It compares the region sequenceIds with oldestFlushing and oldestUnFlushed entries.
+   * If a region's entries are larger than min of (oldestFlushing, oldestUnFlushed), then the
+   * region should be flushed before archiving this WAL.
+  */
+  @Test
+  public void testAllRegionsFlushed() {
+    Map<byte[], Long> oldestFlushingSeqNo = new HashMap<byte[], Long>();
+    Map<byte[], Long> oldestUnFlushedSeqNo = new HashMap<byte[], Long>();
+    Map<byte[], Long> seqNo = new HashMap<byte[], Long>();
+    // create a table
+    TableName t1 = TableName.valueOf("t1");
+    // create a region
+    HRegionInfo hri1 = new HRegionInfo(t1, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
+    // variables to mock region sequenceIds
+    final AtomicLong sequenceId1 = new AtomicLong(1);
+    // test empty map
+    assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+    // add entries in the region
+    seqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.incrementAndGet());
+    oldestUnFlushedSeqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.get());
+    // should say region1 is not flushed.
+    assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+    // test with entries in oldestFlushing map.
+    oldestUnFlushedSeqNo.clear();
+    oldestFlushingSeqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.get());
+    assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+    // simulate region flush, i.e., clear oldestFlushing and oldestUnflushed maps
+    oldestFlushingSeqNo.clear();
+    oldestUnFlushedSeqNo.clear();
+    assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+    // insert some large values for region1
+    oldestUnFlushedSeqNo.put(hri1.getEncodedNameAsBytes(), 1000l);
+    seqNo.put(hri1.getEncodedNameAsBytes(), 1500l);
+    assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+
+    // tests when oldestUnFlushed/oldestFlushing contains larger value.
+    // It means region is flushed.
+    oldestFlushingSeqNo.put(hri1.getEncodedNameAsBytes(), 1200l);
+    oldestUnFlushedSeqNo.clear();
+    seqNo.put(hri1.getEncodedNameAsBytes(), 1199l);
+    assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo));
+  }
+
+  /**
+   * helper method to simulate region flush for a WAL.
+   * @param hlog
+   * @param regionEncodedName
+   */
+  private void flushRegion(HLog hlog, byte[] regionEncodedName) {
+    hlog.startCacheFlush(regionEncodedName);
+    hlog.completeCacheFlush(regionEncodedName);
+  }
+
   static class DumbWALActionsListener implements WALActionsListener {
     int increments = 0;
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java Thu Nov  7 18:10:38 2013
@@ -1089,6 +1089,7 @@ public class TestHLogSplit {
       HRegionInfo regioninfo = new HRegionInfo(tableName,
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
       log = HLogFactory.createHLog(fs, HBASEDIR, logName, conf);
+      final AtomicLong sequenceId = new AtomicLong(1);
 
       final int total = 20;
       for (int i = 0; i < total; i++) {
@@ -1096,7 +1097,7 @@ public class TestHLogSplit {
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
         HTableDescriptor htd = new HTableDescriptor(tableName);
         htd.addFamily(new HColumnDescriptor("column"));
-        log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd);
+        log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
       }
       // Send the data to HDFS datanodes and close the HDFS writer
       log.sync();

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java Thu Nov  7 18:10:38 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionse
 import static org.junit.Assert.assertFalse;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -110,6 +111,7 @@ public class TestLogRollingNoCluster {
     @Override
     public void run() {
       this.log.info(getName() +" started");
+      final AtomicLong sequenceId = new AtomicLong(1);
       try {
         for (int i = 0; i < this.count; i++) {
           long now = System.currentTimeMillis();
@@ -123,7 +125,7 @@ public class TestLogRollingNoCluster {
 
           this.wal.append(HRegionInfo.FIRST_META_REGIONINFO,
               HTableDescriptor.META_TABLEDESC.getTableName(),
-              edit, now, HTableDescriptor.META_TABLEDESC);
+              edit, now, HTableDescriptor.META_TABLEDESC, sequenceId);
         }
         String msg = getName() + " finished";
         if (isException())

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java?rev=1539743&r1=1539742&r2=1539743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java Thu Nov  7 18:10:38 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionse
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -89,6 +90,7 @@ public class TestWALActionsListener {
     DummyWALActionsListener laterobserver = new DummyWALActionsListener();
     HLog hlog = HLogFactory.createHLog(fs, TEST_UTIL.getDataTestDir(), logName,
                                        conf, list, null);
+    final AtomicLong sequenceId = new AtomicLong(1);
     HRegionInfo hri = new HRegionInfo(TableName.valueOf(SOME_BYTES),
              SOME_BYTES, SOME_BYTES, false);
 
@@ -100,7 +102,7 @@ public class TestWALActionsListener {
       HTableDescriptor htd = new HTableDescriptor();
       htd.addFamily(new HColumnDescriptor(b));
 
-      hlog.append(hri, TableName.valueOf(b), edit, 0, htd);
+      hlog.append(hri, TableName.valueOf(b), edit, 0, htd, sequenceId);
       if (i == 10) {
         hlog.registerWALActionsListener(laterobserver);
       }