[1/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

Repository: hbase
Updated Branches:
  refs/heads/hbase-11339 6388b3baf -> b31a6acf4


http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
new file mode 100644
index 0000000..0b8b658
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreCompaction.java
@@ -0,0 +1,460 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
+import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+/**
+ * Test mob store compaction
+ */
+@Category(MediumTests.class)
+public class TestMobStoreCompaction {
+  @Rule
+  public TestName name = new TestName();
+  static final Log LOG = LogFactory.getLog(TestMobStoreCompaction.class.getName());
+  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+  private Configuration conf = null;
+
+  private HRegion region = null;
+  private HTableDescriptor htd = null;
+  private HColumnDescriptor hcd = null;
+  private long mobCellThreshold = 1000;
+
+  private FileSystem fs;
+
+  private static final byte[] COLUMN_FAMILY = fam1;
+  private final byte[] STARTROW = Bytes.toBytes(START_KEY);
+  private int compactionThreshold;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+    UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    UTIL.shutdownMiniCluster();
+  }
+
+  private void init(Configuration conf, long mobThreshold) throws Exception {
+    this.conf = conf;
+    this.mobCellThreshold = mobThreshold;
+    HBaseTestingUtility UTIL = new HBaseTestingUtility(conf);
+
+    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
+    htd = UTIL.createTableDescriptor(name.getMethodName());
+    hcd = new HColumnDescriptor(COLUMN_FAMILY);
+    hcd.setMobEnabled(true);
+    hcd.setMobThreshold(mobThreshold);
+    hcd.setMaxVersions(1);
+    htd.modifyFamily(hcd);
+
+    region = UTIL.createLocalHRegion(htd, null, null);
+    fs = FileSystem.get(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    region.close();
+    fs.delete(UTIL.getDataTestDir(), true);
+  }
+
+  /**
+   * During compaction, cells smaller than the threshold won't be affected.
+   */
+  @Test
+  public void testSmallerValue() throws Exception {
+    init(UTIL.getConfiguration(), 500);
+    byte[] dummyData = makeDummyData(300); // smaller than mob threshold
+    HRegionIncommon loader = new HRegionIncommon(region);
+    // one hfile per row
+    for (int i = 0; i < compactionThreshold; i++) {
+      Put p = createPut(i, dummyData);
+      loader.put(p);
+      loader.flushcache();
+    }
+    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
+    assertEquals("Before compaction: mob file count", 0, countMobFiles());
+    assertEquals("Before compaction: rows", compactionThreshold, countRows());
+    assertEquals("Before compaction: mob rows", 0, countMobRows());
+
+    region.compactStores();
+
+    assertEquals("After compaction: store files", 1, countStoreFiles());
+    assertEquals("After compaction: mob file count", 0, countMobFiles());
+    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
+    assertEquals("After compaction: rows", compactionThreshold, countRows());
+    assertEquals("After compaction: mob rows", 0, countMobRows());
+  }
+
+  /**
+   * During compaction, the mob threshold size is changed.
+   */
+  @Test
+  public void testLargerValue() throws Exception {
+    init(UTIL.getConfiguration(), 200);
+    byte[] dummyData = makeDummyData(300); // larger than mob threshold
+    HRegionIncommon loader = new HRegionIncommon(region);
+    for (int i = 0; i < compactionThreshold; i++) {
+      Put p = createPut(i, dummyData);
+      loader.put(p);
+      loader.flushcache();
+    }
+    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
+    assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
+    assertEquals("Before compaction: rows", compactionThreshold, countRows());
+    assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
+    assertEquals("Before compaction: number of mob cells", compactionThreshold,
+        countMobCellsInMetadata());
+    // Change the threshold larger than the data size
+    region.getTableDesc().getFamily(COLUMN_FAMILY).setMobThreshold(500);
+    region.initialize();
+    region.compactStores();
+
+    assertEquals("After compaction: store files", 1, countStoreFiles());
+    assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
+    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
+    assertEquals("After compaction: rows", compactionThreshold, countRows());
+    assertEquals("After compaction: mob rows", 0, countMobRows());
+  }
+
+  /**
+   * This test will first generate store files, then bulk load them and trigger the compaction.
+   * When compaction, the cell value will be larger than the threshold.
+   */
+  @Test
+  public void testMobCompactionWithBulkload() throws Exception {
+    // The following will produce store files of 600.
+    init(UTIL.getConfiguration(), 300);
+    byte[] dummyData = makeDummyData(600);
+
+    Path hbaseRootDir = FSUtils.getRootDir(conf);
+    Path basedir = new Path(hbaseRootDir, htd.getNameAsString());
+    List<Pair<byte[], String>> hfiles = new ArrayList<>(1);
+    for (int i = 0; i < compactionThreshold; i++) {
+      Path hpath = new Path(basedir, "hfile" + i);
+      hfiles.add(Pair.newPair(COLUMN_FAMILY, hpath.toString()));
+      createHFile(hpath, i, dummyData);
+    }
+
+    // The following will bulk load the above generated store files and compact, with 600(fileSize)
+    // > 300(threshold)
+    boolean result = region.bulkLoadHFiles(hfiles, true, null);
+    assertTrue("Bulkload result:", result);
+    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
+    assertEquals("Before compaction: mob file count", 0, countMobFiles());
+    assertEquals("Before compaction: rows", compactionThreshold, countRows());
+    assertEquals("Before compaction: mob rows", 0, countMobRows());
+    assertEquals("Before compaction: referenced mob file count", 0, countReferencedMobFiles());
+
+    region.compactStores();
+
+    assertEquals("After compaction: store files", 1, countStoreFiles());
+    assertEquals("After compaction: mob file count:", 1, countMobFiles());
+    assertEquals("After compaction: rows", compactionThreshold, countRows());
+    assertEquals("After compaction: mob rows", compactionThreshold, countMobRows());
+    assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles());
+    assertEquals("After compaction: number of mob cells", compactionThreshold,
+        countMobCellsInMetadata());
+  }
+
+  @Test
+  public void testMajorCompactionAfterDelete() throws Exception {
+    init(UTIL.getConfiguration(), 100);
+    byte[] dummyData = makeDummyData(200); // larger than mob threshold
+    HRegionIncommon loader = new HRegionIncommon(region);
+    // create hfiles and mob hfiles but don't trigger compaction
+    int numHfiles = compactionThreshold - 1;
+    byte[] deleteRow = Bytes.add(STARTROW, Bytes.toBytes(0));
+    for (int i = 0; i < numHfiles; i++) {
+      Put p = createPut(i, dummyData);
+      loader.put(p);
+      loader.flushcache();
+    }
+    assertEquals("Before compaction: store files", numHfiles, countStoreFiles());
+    assertEquals("Before compaction: mob file count", numHfiles, countMobFiles());
+    assertEquals("Before compaction: rows", numHfiles, countRows());
+    assertEquals("Before compaction: mob rows", numHfiles, countMobRows());
+    assertEquals("Before compaction: number of mob cells", numHfiles, countMobCellsInMetadata());
+    // now let's delete some cells that contain mobs
+    Delete delete = new Delete(deleteRow);
+    delete.addFamily(COLUMN_FAMILY);
+    region.delete(delete);
+    loader.flushcache();
+
+    assertEquals("Before compaction: store files", numHfiles + 1, countStoreFiles());
+    assertEquals("Before compaction: mob files", numHfiles, countMobFiles());
+    // region.compactStores();
+    region.compact(true);
+    assertEquals("After compaction: store files", 1, countStoreFiles());
+    // still have original mob hfiles and now added a mob del file
+    assertEquals("After compaction: mob files", numHfiles + 1, countMobFiles());
+
+    Scan scan = new Scan();
+    scan.setRaw(true);
+    InternalScanner scanner = region.getScanner(scan);
+    List<Cell> results = new ArrayList<>();
+    scanner.next(results);
+    int deleteCount = 0;
+    while (!results.isEmpty()) {
+      for (Cell c : results) {
+        if (c.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()) {
+          deleteCount++;
+          assertTrue(Bytes.equals(CellUtil.cloneRow(c), deleteRow));
+        }
+      }
+      results.clear();
+      scanner.next(results);
+    }
+    // assert the delete mark is not retained after the major compaction
+    assertEquals(0, deleteCount);
+    scanner.close();
+    // assert the deleted cell is not counted
+    assertEquals("The cells in mob files", numHfiles - 1, countMobCellsInMobFiles(1));
+  }
+
+  private int countStoreFiles() throws IOException {
+    Store store = region.getStore(COLUMN_FAMILY);
+    return store.getStorefilesCount();
+  }
+
+  private int countMobFiles() throws IOException {
+    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
+        hcd.getNameAsString());
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
+      return files.length;
+    }
+    return 0;
+  }
+
+  private long countMobCellsInMetadata() throws IOException {
+    long mobCellsCount = 0;
+    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
+        hcd.getNameAsString());
+    Configuration copyOfConf = new Configuration(conf);
+    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
+    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
+      for (FileStatus file : files) {
+        StoreFile sf = new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE);
+        Map<byte[], byte[]> fileInfo = sf.createReader().loadFileInfo();
+        byte[] count = fileInfo.get(StoreFile.MOB_CELLS_COUNT);
+        assertTrue(count != null);
+        mobCellsCount += Bytes.toLong(count);
+      }
+    }
+    return mobCellsCount;
+  }
+
+  private Put createPut(int rowIdx, byte[] dummyData) throws IOException {
+    Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)));
+    p.setDurability(Durability.SKIP_WAL);
+    p.addColumn(COLUMN_FAMILY, Bytes.toBytes("colX"), dummyData);
+    return p;
+  }
+
+  /**
+   * Create an HFile with the given number of bytes
+   */
+  private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException {
+    HFileContext meta = new HFileContextBuilder().build();
+    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
+        .withFileContext(meta).create();
+    long now = System.currentTimeMillis();
+    try {
+      KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY,
+          Bytes.toBytes("colX"), now, dummyData);
+      writer.append(kv);
+    } finally {
+      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
+      writer.close();
+    }
+  }
+
+  private int countMobRows() throws IOException {
+    Scan scan = new Scan();
+    // Do not retrieve the mob data when scanning
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    InternalScanner scanner = region.getScanner(scan);
+
+    int scannedCount = 0;
+    List<Cell> results = new ArrayList<>();
+    boolean hasMore = true;
+    while (hasMore) {
+      hasMore = scanner.next(results);
+      for (Cell c : results) {
+        if (MobUtils.isMobReferenceCell(c)) {
+          scannedCount++;
+        }
+      }
+      results.clear();
+    }
+    scanner.close();
+
+    return scannedCount;
+  }
+
+  private int countRows() throws IOException {
+    Scan scan = new Scan();
+    // Do not retrieve the mob data when scanning
+    InternalScanner scanner = region.getScanner(scan);
+
+    int scannedCount = 0;
+    List<Cell> results = new ArrayList<Cell>();
+    boolean hasMore = true;
+    while (hasMore) {
+      hasMore = scanner.next(results);
+      scannedCount += results.size();
+      results.clear();
+    }
+    scanner.close();
+
+    return scannedCount;
+  }
+
+  private byte[] makeDummyData(int size) {
+    byte[] dummyData = new byte[size];
+    new Random().nextBytes(dummyData);
+    return dummyData;
+  }
+
+  private int countReferencedMobFiles() throws IOException {
+    Scan scan = new Scan();
+    // Do not retrieve the mob data when scanning
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    InternalScanner scanner = region.getScanner(scan);
+
+    List<Cell> kvs = new ArrayList<>();
+    boolean hasMore = true;
+    String fileName;
+    Set<String> files = new HashSet<>();
+    do {
+      kvs.clear();
+      hasMore = scanner.next(kvs);
+      for (Cell kv : kvs) {
+        if (!MobUtils.isMobReferenceCell(kv)) {
+          continue;
+        }
+        if (!MobUtils.hasValidMobRefCellValue(kv)) {
+          continue;
+        }
+        int size = MobUtils.getMobValueLength(kv);
+        if (size <= mobCellThreshold) {
+          continue;
+        }
+        fileName = MobUtils.getMobFileName(kv);
+        if (fileName.isEmpty()) {
+          continue;
+        }
+        files.add(fileName);
+        Path familyPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(),
+            hcd.getNameAsString());
+        assertTrue(fs.exists(new Path(familyPath, fileName)));
+      }
+    } while (hasMore);
+
+    scanner.close();
+
+    return files.size();
+  }
+
+  private int countMobCellsInMobFiles(int expectedNumDelfiles) throws IOException {
+    Configuration copyOfConf = new Configuration(conf);
+    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
+    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
+    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
+        hcd.getNameAsString());
+    List<StoreFile> sfs = new ArrayList<>();
+    int numDelfiles = 0;
+    int size = 0;
+    if (fs.exists(mobDirPath)) {
+      for (FileStatus f : fs.listStatus(mobDirPath)) {
+        StoreFile sf = new StoreFile(fs, f.getPath(), conf, cacheConfig, BloomType.NONE);
+        sfs.add(sf);
+        if (StoreFileInfo.isDelFile(sf.getPath())) {
+          numDelfiles++;
+        }
+      }
+      List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, null,
+          HConstants.LATEST_TIMESTAMP);
+      Scan scan = new Scan();
+      scan.setMaxVersions(hcd.getMaxVersions());
+      long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
+      long ttl = HStore.determineTTLFromFamily(hcd);
+      ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
+      StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
+          scanners, 0L, HConstants.LATEST_TIMESTAMP);
+      List<Cell> results = new ArrayList<>();
+      boolean hasMore = true;
+      while (hasMore) {
+        hasMore = scanner.next(results);
+        size += results.size();
+        results.clear();
+      }
+    }
+    // assert the number of the existing del files
+    assertEquals(expectedNumDelfiles, numDelfiles);
+    return size;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 6fa708f..1466acc 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -998,7 +998,7 @@ module Hbase
     # Requests a mob file compaction
     def compact_mob(table_name, family = nil)
       if family == nil
-        @admin.compactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name))
+        @admin.compactMobs(org.apache.hadoop.hbase.TableName.valueOf(table_name))
       else
         # We are compacting a mob column family within a table.
         @admin.compactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes)
@@ -1009,7 +1009,7 @@ module Hbase
     # Requests a mob file major compaction
     def major_compact_mob(table_name, family = nil)
       if family == nil
-        @admin.majorCompactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name))
+        @admin.majorCompactMobs(org.apache.hadoop.hbase.TableName.valueOf(table_name))
       else
         # We are major compacting a mob column family within a table.
         @admin.majorCompactMob(org.apache.hadoop.hbase.TableName.valueOf(table_name), family.to_java_bytes)

[5/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
index 464a0e7..dd33cda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobConstants.java
@@ -77,43 +77,43 @@ public class MobConstants {
   public final static String EMPTY_STRING = "";
   /**
    * If the size of a mob file is less than this value, it's regarded as a small file and needs to
-   * be merged in mob file compaction. The default value is 192MB.
+   * be merged in mob compaction. The default value is 192MB.
    */
-  public static final String MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD =
-    "hbase.mob.file.compaction.mergeable.threshold";
-  public static final long DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD = 192 * 1024 * 1024;
+  public static final String MOB_COMPACTION_MERGEABLE_THRESHOLD =
+    "hbase.mob.compaction.mergeable.threshold";
+  public static final long DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD = 192 * 1024 * 1024;
   /**
-   * The max number of del files that is allowed in the mob file compaction. In the mob file
+   * The max number of del files that is allowed in the mob file compaction. In the mob
    * compaction, when the number of existing del files is larger than this value, they are merged
    * until number of del files is not larger this value. The default value is 3.
    */
   public static final String MOB_DELFILE_MAX_COUNT = "hbase.mob.delfile.max.count";
   public static final int DEFAULT_MOB_DELFILE_MAX_COUNT = 3;
   /**
-   * The max number of the mob files that is allowed in a batch of the mob file compaction.
-   * The mob file compaction merges the small mob files to bigger ones. If the number of the
+   * The max number of the mob files that is allowed in a batch of the mob compaction.
+   * The mob compaction merges the small mob files to bigger ones. If the number of the
    * small files is very large, it could lead to a "too many opened file handlers" in the merge.
    * And the merge has to be split into batches. This value limits the number of mob files
-   * that are selected in a batch of the mob file compaction. The default value is 100.
+   * that are selected in a batch of the mob compaction. The default value is 100.
    */
-  public static final String MOB_FILE_COMPACTION_BATCH_SIZE =
-    "hbase.mob.file.compaction.batch.size";
-  public static final int DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE = 100;
+  public static final String MOB_COMPACTION_BATCH_SIZE =
+    "hbase.mob.compaction.batch.size";
+  public static final int DEFAULT_MOB_COMPACTION_BATCH_SIZE = 100;
   /**
-   * The period that MobFileCompactionChore runs. The unit is millisecond.
+   * The period that MobCompactionChore runs. The unit is second.
    * The default value is one week.
    */
-  public static final String MOB_FILE_COMPACTION_CHORE_PERIOD =
-    "hbase.mob.file.compaction.chore.period";
-  public static final int DEFAULT_MOB_FILE_COMPACTION_CHORE_PERIOD =
+  public static final String MOB_COMPACTION_CHORE_PERIOD =
+    "hbase.mob.compaction.chore.period";
+  public static final int DEFAULT_MOB_COMPACTION_CHORE_PERIOD =
     24 * 60 * 60 * 7; // a week
-  public static final String MOB_FILE_COMPACTOR_CLASS_KEY = "hbase.mob.file.compactor.class";
+  public static final String MOB_COMPACTOR_CLASS_KEY = "hbase.mob.compactor.class";
   /**
-   * The max number of threads used in MobFileCompactor.
+   * The max number of threads used in MobCompactor.
    */
-  public static final String MOB_FILE_COMPACTION_THREADS_MAX =
-    "hbase.mob.file.compaction.threads.max";
-  public static final int DEFAULT_MOB_FILE_COMPACTION_THREADS_MAX = 1;
+  public static final String MOB_COMPACTION_THREADS_MAX =
+    "hbase.mob.compaction.threads.max";
+  public static final int DEFAULT_MOB_COMPACTION_THREADS_MAX = 1;
   private MobConstants() {
 
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java
index 7d8c9a5..0780f87 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java
@@ -177,7 +177,7 @@ public class MobFileCache {
           evictedFileCount.incrementAndGet();
         }
       } catch (IOException e) {
-        LOG.error("Fail to evict the file " + fileName, e);
+        LOG.error("Failed to evict the file " + fileName, e);
       } finally {
         if (lockEntry != null) {
           keyLock.releaseLockEntry(lockEntry);
@@ -249,7 +249,7 @@ public class MobFileCache {
 
   public void shutdown() {
     this.scheduleThreadPool.shutdown();
-    for (int i = 0; i < 10; i++) {
+    for (int i = 0; i < 100; i++) {
       if (!this.scheduleThreadPool.isShutdown()) {
         try {
           Thread.sleep(10);

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileName.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileName.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileName.java
index 937e965..796fe4d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileName.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileName.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.hbase.util.MD5Hash;
  * It consists of a md5 of a start key, a date and an uuid.
  * It looks like md5(start) + date + uuid.
  * <ol>
- * <li>0-31 characters: md5 hex string of a start key. Since the length of the start key is not
+ * <li>characters 0-31: md5 hex string of a start key. Since the length of the start key is not
  * fixed, have to use the md5 instead which has a fix length.</li>
- * <li>32-39 characters: a string of a date with format yyyymmdd. The date is the latest timestamp
+ * <li>characters 32-39: a string of a date with format yyyymmdd. The date is the latest timestamp
  * of cells in this file</li>
  * <li>the remaining characters: the uuid.</li>
  * </ol>

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
index 2d5f1ad..a54660c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobStoreEngine.java
@@ -43,6 +43,6 @@ public class MobStoreEngine extends DefaultStoreEngine {
    */
   @Override
   protected void createCompactor(Configuration conf, Store store) throws IOException {
-    compactor = new DefaultMobCompactor(conf, store);
+    compactor = new DefaultMobStoreCompactor(conf, store);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
index bbdc47a..c40767c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.CellComparator;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.Tag;
@@ -63,8 +64,8 @@ import org.apache.hadoop.hbase.io.hfile.HFileContext;
 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
 import org.apache.hadoop.hbase.master.TableLockManager;
 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
-import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactor;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactor;
+import org.apache.hadoop.hbase.mob.compactions.MobCompactor;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.HStore;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -249,7 +250,7 @@ public class MobUtils {
     try {
       stats = fs.listStatus(path);
     } catch (FileNotFoundException e) {
-      LOG.warn("Fail to find the mob file " + path, e);
+      LOG.warn("Failed to find the mob file " + path, e);
     }
     if (null == stats) {
       // no file found
@@ -287,7 +288,7 @@ public class MobUtils {
             filesToClean);
         deletedFileCount = filesToClean.size();
       } catch (IOException e) {
-        LOG.error("Fail to delete the mob files " + filesToClean, e);
+        LOG.error("Failed to delete the mob files " + filesToClean, e);
       }
     }
     LOG.info(deletedFileCount + " expired mob files are deleted");
@@ -555,7 +556,7 @@ public class MobUtils {
   }
 
   /**
-   * Creates a writer for the del file in temp directory.
+   * Creates a writer for the mob file in temp directory.
    * @param conf The current configuration.
    * @param fs The current file system.
    * @param family The descriptor of the current column family.
@@ -631,7 +632,7 @@ public class MobUtils {
       storeFile = new StoreFile(fs, path, conf, cacheConfig, BloomType.NONE);
       storeFile.createReader();
     } catch (IOException e) {
-      LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e);
+      LOG.error("Failed to open mob file[" + path + "], keep it in temp directory.", e);
       throw e;
     } finally {
       if (storeFile != null) {
@@ -692,22 +693,22 @@ public class MobUtils {
   }
 
   /**
-   * Performs the mob file compaction.
+   * Performs the mob compaction.
    * @param conf the Configuration
    * @param fs the file system
    * @param tableName the table the compact
    * @param hcd the column descriptor
    * @param pool the thread pool
    * @param tableLockManager the tableLock manager
-   * @param isForceAllFiles Whether add all mob files into the compaction.
+   * @param allFiles Whether add all mob files into the compaction.
    */
-  public static void doMobFileCompaction(Configuration conf, FileSystem fs, TableName tableName,
+  public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
     HColumnDescriptor hcd, ExecutorService pool, TableLockManager tableLockManager,
-    boolean isForceAllFiles) throws IOException {
-    String className = conf.get(MobConstants.MOB_FILE_COMPACTOR_CLASS_KEY,
-      PartitionedMobFileCompactor.class.getName());
-    // instantiate the mob file compactor.
-    MobFileCompactor compactor = null;
+    boolean allFiles) throws IOException {
+    String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY,
+      PartitionedMobCompactor.class.getName());
+    // instantiate the mob compactor.
+    MobCompactor compactor = null;
     try {
       compactor = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] {
         Configuration.class, FileSystem.class, TableName.class, HColumnDescriptor.class,
@@ -724,21 +725,21 @@ public class MobUtils {
       // the tableLockManager might be null in testing. In that case, it is lock-free.
       if (tableLockManager != null) {
         lock = tableLockManager.writeLock(MobUtils.getTableLockName(tableName),
-          "Run MobFileCompaction");
+          "Run MobCompactor");
         lock.acquire();
       }
       tableLocked = true;
-      compactor.compact(isForceAllFiles);
+      compactor.compact(allFiles);
     } catch (Exception e) {
-      LOG.error("Fail to compact the mob files for the column " + hcd.getNameAsString()
+      LOG.error("Failed to compact the mob files for the column " + hcd.getNameAsString()
         + " in the table " + tableName.getNameAsString(), e);
     } finally {
       if (lock != null && tableLocked) {
         try {
           lock.release();
         } catch (IOException e) {
-          LOG.error("Fail to release the write lock for the table " + tableName.getNameAsString(),
-            e);
+          LOG.error(
+            "Failed to release the write lock for the table " + tableName.getNameAsString(), e);
         }
       }
     }
@@ -749,15 +750,15 @@ public class MobUtils {
    * @param conf the Configuration
    * @return A thread pool.
    */
-  public static ExecutorService createMobFileCompactorThreadPool(Configuration conf) {
-    int maxThreads = conf.getInt(MobConstants.MOB_FILE_COMPACTION_THREADS_MAX,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_THREADS_MAX);
+  public static ExecutorService createMobCompactorThreadPool(Configuration conf) {
+    int maxThreads = conf.getInt(MobConstants.MOB_COMPACTION_THREADS_MAX,
+      MobConstants.DEFAULT_MOB_COMPACTION_THREADS_MAX);
     if (maxThreads == 0) {
       maxThreads = 1;
     }
     final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
     ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, queue,
-      Threads.newDaemonThreadFactory("MobFileCompactor"), new RejectedExecutionHandler() {
+      Threads.newDaemonThreadFactory("MobCompactor"), new RejectedExecutionHandler() {
         @Override
         public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
           try {
@@ -839,4 +840,19 @@ public class MobUtils {
     }
     return cryptoContext;
   }
+
+  /**
+   * Checks whether this table has mob-enabled columns.
+   * @param htd The current table descriptor.
+   * @return Whether this table has mob-enabled columns.
+   */
+  public static boolean hasMobColumns(HTableDescriptor htd) {
+    HColumnDescriptor[] hcds = htd.getColumnFamilies();
+    for (HColumnDescriptor hcd : hcds) {
+      if (hcd.isMobEnabled()) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactionRequest.java
new file mode 100644
index 0000000..5d162b4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactionRequest.java
@@ -0,0 +1,64 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * The compaction request for mob files.
+ */
+@InterfaceAudience.Private
+public abstract class MobCompactionRequest {
+
+  protected long selectionTime;
+  protected CompactionType type = CompactionType.PART_FILES;
+
+  public void setCompactionType(CompactionType type) {
+    this.type = type;
+  }
+
+  /**
+   * Gets the selection time.
+   * @return The selection time.
+   */
+  public long getSelectionTime() {
+    return this.selectionTime;
+  }
+
+  /**
+   * Gets the compaction type.
+   * @return The compaction type.
+   */
+  public CompactionType getCompactionType() {
+    return type;
+  }
+
+  protected enum CompactionType {
+
+    /**
+     * Part of mob files are selected.
+     */
+    PART_FILES,
+
+    /**
+     * All of mob files are selected.
+     */
+    ALL_FILES;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactor.java
new file mode 100644
index 0000000..156c6f6
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/MobCompactor.java
@@ -0,0 +1,90 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * A mob compactor to directly compact the mob files.
+ */
+@InterfaceAudience.Private
+public abstract class MobCompactor {
+
+  protected FileSystem fs;
+  protected Configuration conf;
+  protected TableName tableName;
+  protected HColumnDescriptor column;
+
+  protected Path mobTableDir;
+  protected Path mobFamilyDir;
+  protected ExecutorService pool;
+
+  public MobCompactor(Configuration conf, FileSystem fs, TableName tableName,
+    HColumnDescriptor column, ExecutorService pool) {
+    this.conf = conf;
+    this.fs = fs;
+    this.tableName = tableName;
+    this.column = column;
+    this.pool = pool;
+    mobTableDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);
+    mobFamilyDir = MobUtils.getMobFamilyPath(conf, tableName, column.getNameAsString());
+  }
+
+  /**
+   * Compacts the mob files for the current column family.
+   * @return The paths of new mob files generated in the compaction.
+   * @throws IOException
+   */
+  public List<Path> compact() throws IOException {
+    return compact(false);
+  }
+
+  /**
+   * Compacts the mob files by compaction type for the current column family.
+   * @param allFiles Whether add all mob files into the compaction.
+   * @return The paths of new mob files generated in the compaction.
+   * @throws IOException
+   */
+  public List<Path> compact(boolean allFiles) throws IOException {
+    return compact(Arrays.asList(fs.listStatus(mobFamilyDir)), allFiles);
+  }
+
+  /**
+   * Compacts the candidate mob files.
+   * @param files The candidate mob files.
+   * @param allFiles Whether add all mob files into the compaction.
+   * @return The paths of new mob files generated in the compaction.
+   * @throws IOException
+   */
+  public abstract List<Path> compact(List<FileStatus> files, boolean allFiles)
+    throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
new file mode 100644
index 0000000..af1eb4a
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
@@ -0,0 +1,146 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * An implementation of {@link MobCompactionRequest} that is used in
+ * {@link PartitionedMobCompactor}.
+ * The mob files that have the same start key and date in their names belong to
+ * the same partition.
+ */
+@InterfaceAudience.Private
+public class PartitionedMobCompactionRequest extends MobCompactionRequest {
+
+  protected Collection<FileStatus> delFiles;
+  protected Collection<CompactionPartition> compactionPartitions;
+
+  public PartitionedMobCompactionRequest(Collection<CompactionPartition> compactionPartitions,
+    Collection<FileStatus> delFiles) {
+    this.selectionTime = EnvironmentEdgeManager.currentTime();
+    this.compactionPartitions = compactionPartitions;
+    this.delFiles = delFiles;
+  }
+
+  /**
+   * Gets the compaction partitions.
+   * @return The compaction partitions.
+   */
+  public Collection<CompactionPartition> getCompactionPartitions() {
+    return this.compactionPartitions;
+  }
+
+  /**
+   * Gets the del files.
+   * @return The del files.
+   */
+  public Collection<FileStatus> getDelFiles() {
+    return this.delFiles;
+  }
+
+  /**
+   * The partition in the mob compaction.
+   * The mob files that have the same start key and date in their names belong to
+   * the same partition.
+   */
+  protected static class CompactionPartition {
+    private List<FileStatus> files = new ArrayList<FileStatus>();
+    private CompactionPartitionId partitionId;
+
+    public CompactionPartition(CompactionPartitionId partitionId) {
+      this.partitionId = partitionId;
+    }
+
+    public CompactionPartitionId getPartitionId() {
+      return this.partitionId;
+    }
+
+    public void addFile(FileStatus file) {
+      files.add(file);
+    }
+
+    public List<FileStatus> listFiles() {
+      return Collections.unmodifiableList(files);
+    }
+  }
+
+  /**
+   * The partition id that consists of start key and date of the mob file name.
+   */
+  public static class CompactionPartitionId {
+
+    private String startKey;
+    private String date;
+
+    public CompactionPartitionId(String startKey, String date) {
+      if (startKey == null || date == null) {
+        throw new IllegalArgumentException("Neither of start key and date could be null");
+      }
+      this.startKey = startKey;
+      this.date = date;
+    }
+
+    public String getStartKey() {
+      return this.startKey;
+    }
+
+    public String getDate() {
+      return this.date;
+    }
+
+    @Override
+    public int hashCode() {
+      int result = 17;
+      result = 31 * result + startKey.hashCode();
+      result = 31 * result + date.hashCode();
+      return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (!(obj instanceof CompactionPartitionId)) {
+        return false;
+      }
+      CompactionPartitionId another = (CompactionPartitionId) obj;
+      if (!this.startKey.equals(another.startKey)) {
+        return false;
+      }
+      if (!this.date.equals(another.date)) {
+        return false;
+      }
+      return true;
+    }
+
+    @Override
+    public String toString() {
+      return new StringBuilder(startKey).append(date).toString();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
new file mode 100644
index 0000000..065787e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
@@ -0,0 +1,655 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.TagType;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.crypto.Encryption;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobFileName;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartitionId;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.regionserver.ScanInfo;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.regionserver.StoreScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * An implementation of {@link MobCompactor} that compacts the mob files in partitions.
+ */
+@InterfaceAudience.Private
+public class PartitionedMobCompactor extends MobCompactor {
+
+  private static final Log LOG = LogFactory.getLog(PartitionedMobCompactor.class);
+  protected long mergeableSize;
+  protected int delFileMaxCount;
+  /** The number of files compacted in a batch */
+  protected int compactionBatchSize;
+  protected int compactionKVMax;
+
+  private Path tempPath;
+  private Path bulkloadPath;
+  private CacheConfig compactionCacheConfig;
+  private Tag tableNameTag;
+  private Encryption.Context cryptoContext = Encryption.Context.NONE;
+
+  public PartitionedMobCompactor(Configuration conf, FileSystem fs, TableName tableName,
+    HColumnDescriptor column, ExecutorService pool) throws IOException {
+    super(conf, fs, tableName, column, pool);
+    mergeableSize = conf.getLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
+      MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
+    delFileMaxCount = conf.getInt(MobConstants.MOB_DELFILE_MAX_COUNT,
+      MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
+    // default is 100
+    compactionBatchSize = conf.getInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
+      MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
+    tempPath = new Path(MobUtils.getMobHome(conf), MobConstants.TEMP_DIR_NAME);
+    bulkloadPath = new Path(tempPath, new Path(MobConstants.BULKLOAD_DIR_NAME, new Path(
+      tableName.getNamespaceAsString(), tableName.getQualifierAsString())));
+    compactionKVMax = this.conf.getInt(HConstants.COMPACTION_KV_MAX,
+      HConstants.COMPACTION_KV_MAX_DEFAULT);
+    Configuration copyOfConf = new Configuration(conf);
+    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
+    compactionCacheConfig = new CacheConfig(copyOfConf);
+    tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, tableName.getName());
+    cryptoContext = MobUtils.createEncryptionContext(copyOfConf, column);
+  }
+
+  @Override
+  public List<Path> compact(List<FileStatus> files, boolean allFiles) throws IOException {
+    if (files == null || files.isEmpty()) {
+      LOG.info("No candidate mob files");
+      return null;
+    }
+    LOG.info("is allFiles: " + allFiles);
+    // find the files to compact.
+    PartitionedMobCompactionRequest request = select(files, allFiles);
+    // compact the files.
+    return performCompaction(request);
+  }
+
+  /**
+   * Selects the compacted mob/del files.
+   * Iterates the candidates to find out all the del files and small mob files.
+   * @param candidates All the candidates.
+   * @param allFiles Whether add all mob files into the compaction.
+   * @return A compaction request.
+   * @throws IOException
+   */
+  protected PartitionedMobCompactionRequest select(List<FileStatus> candidates,
+    boolean allFiles) throws IOException {
+    Collection<FileStatus> allDelFiles = new ArrayList<FileStatus>();
+    Map<CompactionPartitionId, CompactionPartition> filesToCompact =
+      new HashMap<CompactionPartitionId, CompactionPartition>();
+    int selectedFileCount = 0;
+    int irrelevantFileCount = 0;
+    for (FileStatus file : candidates) {
+      if (!file.isFile()) {
+        irrelevantFileCount++;
+        continue;
+      }
+      // group the del files and small files.
+      FileStatus linkedFile = file;
+      if (HFileLink.isHFileLink(file.getPath())) {
+        HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
+        linkedFile = getLinkedFileStatus(link);
+        if (linkedFile == null) {
+          // If the linked file cannot be found, regard it as an irrelevantFileCount file
+          irrelevantFileCount++;
+          continue;
+        }
+      }
+      if (StoreFileInfo.isDelFile(linkedFile.getPath())) {
+        allDelFiles.add(file);
+      } else if (allFiles || linkedFile.getLen() < mergeableSize) {
+        // add all files if allFiles is true,
+        // otherwise add the small files to the merge pool
+        MobFileName fileName = MobFileName.create(linkedFile.getPath().getName());
+        CompactionPartitionId id = new CompactionPartitionId(fileName.getStartKey(),
+          fileName.getDate());
+        CompactionPartition compactionPartition = filesToCompact.get(id);
+        if (compactionPartition == null) {
+          compactionPartition = new CompactionPartition(id);
+          compactionPartition.addFile(file);
+          filesToCompact.put(id, compactionPartition);
+        } else {
+          compactionPartition.addFile(file);
+        }
+        selectedFileCount++;
+      }
+    }
+    PartitionedMobCompactionRequest request = new PartitionedMobCompactionRequest(
+      filesToCompact.values(), allDelFiles);
+    if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount)) {
+      // all the files are selected
+      request.setCompactionType(CompactionType.ALL_FILES);
+    }
+    LOG.info("The compaction type is " + request.getCompactionType() + ", the request has "
+      + allDelFiles.size() + " del files, " + selectedFileCount + " selected files, and "
+      + irrelevantFileCount + " irrelevant files");
+    return request;
+  }
+
+  /**
+   * Performs the compaction on the selected files.
+   * <ol>
+   * <li>Compacts the del files.</li>
+   * <li>Compacts the selected small mob files and all the del files.</li>
+   * <li>If all the candidates are selected, delete the del files.</li>
+   * </ol>
+   * @param request The compaction request.
+   * @return The paths of new mob files generated in the compaction.
+   * @throws IOException
+   */
+  protected List<Path> performCompaction(PartitionedMobCompactionRequest request)
+    throws IOException {
+    // merge the del files
+    List<Path> delFilePaths = new ArrayList<Path>();
+    for (FileStatus delFile : request.delFiles) {
+      delFilePaths.add(delFile.getPath());
+    }
+    List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
+    List<StoreFile> newDelFiles = new ArrayList<StoreFile>();
+    for (Path newDelPath : newDelPaths) {
+      StoreFile sf = new StoreFile(fs, newDelPath, conf, compactionCacheConfig, BloomType.NONE);
+      newDelFiles.add(sf);
+    }
+    LOG.info("After merging, there are " + newDelFiles.size() + " del files");
+    // compact the mob files by partitions.
+    List<Path> paths = compactMobFiles(request, newDelFiles);
+    LOG.info("After compaction, there are " + paths.size() + " mob files");
+    // archive the del files if all the mob files are selected.
+    if (request.type == CompactionType.ALL_FILES && !newDelPaths.isEmpty()) {
+      LOG.info("After a mob compaction with all files selected, archiving the del files "
+        + newDelPaths);
+      try {
+        MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), newDelFiles);
+      } catch (IOException e) {
+        LOG.error("Failed to archive the del files " + newDelPaths, e);
+      }
+    }
+    return paths;
+  }
+
+  /**
+   * Compacts the selected small mob files and all the del files.
+   * @param request The compaction request.
+   * @param delFiles The del files.
+   * @return The paths of new mob files after compactions.
+   * @throws IOException
+   */
+  protected List<Path> compactMobFiles(final PartitionedMobCompactionRequest request,
+    final List<StoreFile> delFiles) throws IOException {
+    Collection<CompactionPartition> partitions = request.compactionPartitions;
+    if (partitions == null || partitions.isEmpty()) {
+      LOG.info("No partitions of mob files");
+      return Collections.emptyList();
+    }
+    List<Path> paths = new ArrayList<Path>();
+    Connection c = ConnectionFactory.createConnection(conf);
+    final Table table = c.getTable(tableName);
+    try {
+      Map<CompactionPartitionId, Future<List<Path>>> results =
+        new HashMap<CompactionPartitionId, Future<List<Path>>>();
+      // compact the mob files by partitions in parallel.
+      for (final CompactionPartition partition : partitions) {
+        results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {
+          @Override
+          public List<Path> call() throws Exception {
+            LOG.info("Compacting mob files for partition " + partition.getPartitionId());
+            return compactMobFilePartition(request, partition, delFiles, table);
+          }
+        }));
+      }
+      // compact the partitions in parallel.
+      List<CompactionPartitionId> failedPartitions = new ArrayList<CompactionPartitionId>();
+      for (Entry<CompactionPartitionId, Future<List<Path>>> result : results.entrySet()) {
+        try {
+          paths.addAll(result.getValue().get());
+        } catch (Exception e) {
+          // just log the error
+          LOG.error("Failed to compact the partition " + result.getKey(), e);
+          failedPartitions.add(result.getKey());
+        }
+      }
+      if (!failedPartitions.isEmpty()) {
+        // if any partition fails in the compaction, directly throw an exception.
+        throw new IOException("Failed to compact the partitions " + failedPartitions);
+      }
+    } finally {
+      try {
+        table.close();
+      } catch (IOException e) {
+        LOG.error("Failed to close the HTable", e);
+      }
+    }
+    return paths;
+  }
+
+  /**
+   * Compacts a partition of selected small mob files and all the del files.
+   * @param request The compaction request.
+   * @param partition A compaction partition.
+   * @param delFiles The del files.
+   * @param table The current table.
+   * @return The paths of new mob files after compactions.
+   * @throws IOException
+   */
+  private List<Path> compactMobFilePartition(PartitionedMobCompactionRequest request,
+    CompactionPartition partition, List<StoreFile> delFiles, Table table) throws IOException {
+    List<Path> newFiles = new ArrayList<Path>();
+    List<FileStatus> files = partition.listFiles();
+    int offset = 0;
+    Path bulkloadPathOfPartition = new Path(bulkloadPath, partition.getPartitionId().toString());
+    Path bulkloadColumnPath = new Path(bulkloadPathOfPartition, column.getNameAsString());
+    while (offset < files.size()) {
+      int batch = compactionBatchSize;
+      if (files.size() - offset < compactionBatchSize) {
+        batch = files.size() - offset;
+      }
+      if (batch == 1 && delFiles.isEmpty()) {
+        // only one file left and no del files, do not compact it,
+        // and directly add it to the new files.
+        newFiles.add(files.get(offset).getPath());
+        offset++;
+        continue;
+      }
+      // clean the bulkload directory to avoid loading old files.
+      fs.delete(bulkloadPathOfPartition, true);
+      // add the selected mob files and del files into filesToCompact
+      List<StoreFile> filesToCompact = new ArrayList<StoreFile>();
+      for (int i = offset; i < batch + offset; i++) {
+        StoreFile sf = new StoreFile(fs, files.get(i).getPath(), conf, compactionCacheConfig,
+          BloomType.NONE);
+        filesToCompact.add(sf);
+      }
+      filesToCompact.addAll(delFiles);
+      // compact the mob files in a batch.
+      compactMobFilesInBatch(request, partition, table, filesToCompact, batch,
+        bulkloadPathOfPartition, bulkloadColumnPath, newFiles);
+      // move to the next batch.
+      offset += batch;
+    }
+    LOG.info("Compaction is finished. The number of mob files is changed from " + files.size()
+      + " to " + newFiles.size());
+    return newFiles;
+  }
+
+  /**
+   * Compacts a partition of selected small mob files and all the del files in a batch.
+   * @param request The compaction request.
+   * @param partition A compaction partition.
+   * @param table The current table.
+   * @param filesToCompact The files to be compacted.
+   * @param batch The number of mob files to be compacted in a batch.
+   * @param bulkloadPathOfPartition The directory where the bulkload column of the current
+   *        partition is saved.
+   * @param bulkloadColumnPath The directory where the bulkload files of current partition
+   *        are saved.
+   * @param newFiles The paths of new mob files after compactions.
+   * @throws IOException
+   */
+  private void compactMobFilesInBatch(PartitionedMobCompactionRequest request,
+    CompactionPartition partition, Table table, List<StoreFile> filesToCompact, int batch,
+    Path bulkloadPathOfPartition, Path bulkloadColumnPath, List<Path> newFiles)
+    throws IOException {
+    // open scanner to the selected mob files and del files.
+    StoreScanner scanner = createScanner(filesToCompact, ScanType.COMPACT_DROP_DELETES);
+    // the mob files to be compacted, not include the del files.
+    List<StoreFile> mobFilesToCompact = filesToCompact.subList(0, batch);
+    // Pair(maxSeqId, cellsCount)
+    Pair<Long, Long> fileInfo = getFileInfo(mobFilesToCompact);
+    // open writers for the mob files and new ref store files.
+    Writer writer = null;
+    Writer refFileWriter = null;
+    Path filePath = null;
+    Path refFilePath = null;
+    long mobCells = 0;
+    try {
+      writer = MobUtils.createWriter(conf, fs, column, partition.getPartitionId().getDate(),
+        tempPath, Long.MAX_VALUE, column.getCompactionCompression(), partition.getPartitionId()
+          .getStartKey(), compactionCacheConfig, cryptoContext);
+      filePath = writer.getPath();
+      byte[] fileName = Bytes.toBytes(filePath.getName());
+      // create a temp file and open a writer for it in the bulkloadPath
+      refFileWriter = MobUtils.createRefFileWriter(conf, fs, column, bulkloadColumnPath, fileInfo
+        .getSecond().longValue(), compactionCacheConfig, cryptoContext);
+      refFilePath = refFileWriter.getPath();
+      List<Cell> cells = new ArrayList<Cell>();
+      boolean hasMore = false;
+      ScannerContext scannerContext =
+              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
+      do {
+        hasMore = scanner.next(cells, scannerContext);
+        for (Cell cell : cells) {
+          // write the mob cell to the mob file.
+          writer.append(cell);
+          // write the new reference cell to the store file.
+          KeyValue reference = MobUtils.createMobRefKeyValue(cell, fileName, tableNameTag);
+          refFileWriter.append(reference);
+          mobCells++;
+        }
+        cells.clear();
+      } while (hasMore);
+    } finally {
+      // close the scanner.
+      scanner.close();
+      // append metadata to the mob file, and close the mob file writer.
+      closeMobFileWriter(writer, fileInfo.getFirst(), mobCells);
+      // append metadata and bulkload info to the ref mob file, and close the writer.
+      closeRefFileWriter(refFileWriter, fileInfo.getFirst(), request.selectionTime);
+    }
+    if (mobCells > 0) {
+      // commit mob file
+      MobUtils.commitFile(conf, fs, filePath, mobFamilyDir, compactionCacheConfig);
+      // bulkload the ref file
+      bulkloadRefFile(table, bulkloadPathOfPartition, filePath.getName());
+      newFiles.add(new Path(mobFamilyDir, filePath.getName()));
+    } else {
+      // remove the new files
+      // the mob file is empty, delete it instead of committing.
+      deletePath(filePath);
+      // the ref file is empty, delete it instead of committing.
+      deletePath(refFilePath);
+    }
+    // archive the old mob files, do not archive the del files.
+    try {
+      MobUtils
+        .removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), mobFilesToCompact);
+    } catch (IOException e) {
+      LOG.error("Failed to archive the files " + mobFilesToCompact, e);
+    }
+  }
+
+  /**
+   * Compacts the del files in batches which avoids opening too many files.
+   * @param request The compaction request.
+   * @param delFilePaths
+   * @return The paths of new del files after merging or the original files if no merging
+   *         is necessary.
+   * @throws IOException
+   */
+  protected List<Path> compactDelFiles(PartitionedMobCompactionRequest request,
+    List<Path> delFilePaths) throws IOException {
+    if (delFilePaths.size() <= delFileMaxCount) {
+      return delFilePaths;
+    }
+    // when there are more del files than the number that is allowed, merge it firstly.
+    int offset = 0;
+    List<Path> paths = new ArrayList<Path>();
+    while (offset < delFilePaths.size()) {
+      // get the batch
+      int batch = compactionBatchSize;
+      if (delFilePaths.size() - offset < compactionBatchSize) {
+        batch = delFilePaths.size() - offset;
+      }
+      List<StoreFile> batchedDelFiles = new ArrayList<StoreFile>();
+      if (batch == 1) {
+        // only one file left, do not compact it, directly add it to the new files.
+        paths.add(delFilePaths.get(offset));
+        offset++;
+        continue;
+      }
+      for (int i = offset; i < batch + offset; i++) {
+        batchedDelFiles.add(new StoreFile(fs, delFilePaths.get(i), conf, compactionCacheConfig,
+          BloomType.NONE));
+      }
+      // compact the del files in a batch.
+      paths.add(compactDelFilesInBatch(request, batchedDelFiles));
+      // move to the next batch.
+      offset += batch;
+    }
+    return compactDelFiles(request, paths);
+  }
+
+  /**
+   * Compacts the del file in a batch.
+   * @param request The compaction request.
+   * @param delFiles The del files.
+   * @return The path of new del file after merging.
+   * @throws IOException
+   */
+  private Path compactDelFilesInBatch(PartitionedMobCompactionRequest request,
+    List<StoreFile> delFiles) throws IOException {
+    // create a scanner for the del files.
+    StoreScanner scanner = createScanner(delFiles, ScanType.COMPACT_RETAIN_DELETES);
+    Writer writer = null;
+    Path filePath = null;
+    try {
+      writer = MobUtils.createDelFileWriter(conf, fs, column,
+        MobUtils.formatDate(new Date(request.selectionTime)), tempPath, Long.MAX_VALUE,
+        column.getCompactionCompression(), HConstants.EMPTY_START_ROW, compactionCacheConfig,
+        cryptoContext);
+      filePath = writer.getPath();
+      List<Cell> cells = new ArrayList<Cell>();
+      boolean hasMore = false;
+      ScannerContext scannerContext =
+              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
+      do {
+        hasMore = scanner.next(cells, scannerContext);
+        for (Cell cell : cells) {
+          writer.append(cell);
+        }
+        cells.clear();
+      } while (hasMore);
+    } finally {
+      scanner.close();
+      if (writer != null) {
+        try {
+          writer.close();
+        } catch (IOException e) {
+          LOG.error("Failed to close the writer of the file " + filePath, e);
+        }
+      }
+    }
+    // commit the new del file
+    Path path = MobUtils.commitFile(conf, fs, filePath, mobFamilyDir, compactionCacheConfig);
+    // archive the old del files
+    try {
+      MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delFiles);
+    } catch (IOException e) {
+      LOG.error("Failed to archive the old del files " + delFiles, e);
+    }
+    return path;
+  }
+
+  /**
+   * Creates a store scanner.
+   * @param filesToCompact The files to be compacted.
+   * @param scanType The scan type.
+   * @return The store scanner.
+   * @throws IOException
+   */
+  private StoreScanner createScanner(List<StoreFile> filesToCompact, ScanType scanType)
+    throws IOException {
+    List scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, true, false,
+      null, HConstants.LATEST_TIMESTAMP);
+    Scan scan = new Scan();
+    scan.setMaxVersions(column.getMaxVersions());
+    long ttl = HStore.determineTTLFromFamily(column);
+    ScanInfo scanInfo = new ScanInfo(column, ttl, 0, CellComparator.COMPARATOR);
+    StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners, 0L,
+      HConstants.LATEST_TIMESTAMP);
+    return scanner;
+  }
+
+  /**
+   * Bulkloads the current file.
+   * @param table The current table.
+   * @param bulkloadDirectory The path of bulkload directory.
+   * @param fileName The current file name.
+   * @throws IOException
+   */
+  private void bulkloadRefFile(Table table, Path bulkloadDirectory, String fileName)
+    throws IOException {
+    // bulkload the ref file
+    try {
+      LoadIncrementalHFiles bulkload = new LoadIncrementalHFiles(conf);
+      bulkload.doBulkLoad(bulkloadDirectory, (HTable)table);
+    } catch (Exception e) {
+      // delete the committed mob file
+      deletePath(new Path(mobFamilyDir, fileName));
+      throw new IOException(e);
+    } finally {
+      // delete the bulkload files in bulkloadPath
+      deletePath(bulkloadDirectory);
+    }
+  }
+
+  /**
+   * Closes the mob file writer.
+   * @param writer The mob file writer.
+   * @param maxSeqId Maximum sequence id.
+   * @param mobCellsCount The number of mob cells.
+   * @throws IOException
+   */
+  private void closeMobFileWriter(Writer writer, long maxSeqId, long mobCellsCount)
+    throws IOException {
+    if (writer != null) {
+      writer.appendMetadata(maxSeqId, false, mobCellsCount);
+      try {
+        writer.close();
+      } catch (IOException e) {
+        LOG.error("Failed to close the writer of the file " + writer.getPath(), e);
+      }
+    }
+  }
+
+  /**
+   * Closes the ref file writer.
+   * @param writer The ref file writer.
+   * @param maxSeqId Maximum sequence id.
+   * @param bulkloadTime The timestamp at which the bulk load file is created.
+   * @throws IOException
+   */
+  private void closeRefFileWriter(Writer writer, long maxSeqId, long bulkloadTime)
+    throws IOException {
+    if (writer != null) {
+      writer.appendMetadata(maxSeqId, false);
+      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(bulkloadTime));
+      try {
+        writer.close();
+      } catch (IOException e) {
+        LOG.error("Failed to close the writer of the ref file " + writer.getPath(), e);
+      }
+    }
+  }
+
+  /**
+   * Gets the max seqId and number of cells of the store files.
+   * @param storeFiles The store files.
+   * @return The pair of the max seqId and number of cells of the store files.
+   * @throws IOException
+   */
+  private Pair<Long, Long> getFileInfo(List<StoreFile> storeFiles) throws IOException {
+    long maxSeqId = 0;
+    long maxKeyCount = 0;
+    for (StoreFile sf : storeFiles) {
+      // the readers will be closed later after the merge.
+      maxSeqId = Math.max(maxSeqId, sf.getMaxSequenceId());
+      byte[] count = sf.createReader().loadFileInfo().get(StoreFile.MOB_CELLS_COUNT);
+      if (count != null) {
+        maxKeyCount += Bytes.toLong(count);
+      }
+    }
+    return new Pair<Long, Long>(Long.valueOf(maxSeqId), Long.valueOf(maxKeyCount));
+  }
+
+  /**
+   * Deletes a file.
+   * @param path The path of the file to be deleted.
+   */
+  private void deletePath(Path path) {
+    try {
+      if (path != null) {
+        fs.delete(path, true);
+      }
+    } catch (IOException e) {
+      LOG.error("Failed to delete the file " + path, e);
+    }
+  }
+
+  private FileStatus getLinkedFileStatus(HFileLink link) throws IOException {
+    Path[] locations = link.getLocations();
+    for (Path location : locations) {
+      FileStatus file = getFileStatus(location);
+      if (file != null) {
+        return file;
+      }
+    }
+    return null;
+  }
+
+  private FileStatus getFileStatus(Path path) throws IOException {
+    try {
+      if (path != null) {
+        FileStatus file = fs.getFileStatus(path);
+        return file;
+      }
+    } catch (FileNotFoundException e) {
+      LOG.warn("The file " + path + " can not be found", e);
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactionRequest.java
deleted file mode 100644
index 375ba8c..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactionRequest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * The compaction request for mob files.
- */
-@InterfaceAudience.Private
-public abstract class MobFileCompactionRequest {
-
-  protected long selectionTime;
-  protected CompactionType type = CompactionType.PART_FILES;
-
-  public void setCompactionType(CompactionType type) {
-    this.type = type;
-  }
-
-  /**
-   * Gets the selection time.
-   * @return The selection time.
-   */
-  public long getSelectionTime() {
-    return this.selectionTime;
-  }
-
-  /**
-   * Gets the compaction type.
-   * @return The compaction type.
-   */
-  public CompactionType getCompactionType() {
-    return type;
-  }
-
-  protected enum CompactionType {
-
-    /**
-     * Part of mob files are selected.
-     */
-    PART_FILES,
-
-    /**
-     * All of mob files are selected.
-     */
-    ALL_FILES;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactor.java
deleted file mode 100644
index fcb39c5..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/MobFileCompactor.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-/**
- * A mob file compactor to directly compact the mob files.
- */
-@InterfaceAudience.Private
-public abstract class MobFileCompactor {
-
-  protected FileSystem fs;
-  protected Configuration conf;
-  protected TableName tableName;
-  protected HColumnDescriptor column;
-
-  protected Path mobTableDir;
-  protected Path mobFamilyDir;
-  protected ExecutorService pool;
-
-  public MobFileCompactor(Configuration conf, FileSystem fs, TableName tableName,
-    HColumnDescriptor column, ExecutorService pool) {
-    this.conf = conf;
-    this.fs = fs;
-    this.tableName = tableName;
-    this.column = column;
-    this.pool = pool;
-    mobTableDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), tableName);
-    mobFamilyDir = MobUtils.getMobFamilyPath(conf, tableName, column.getNameAsString());
-  }
-
-  /**
-   * Compacts the mob files for the current column family.
-   * @return The paths of new mob files generated in the compaction.
-   * @throws IOException
-   */
-  public List<Path> compact() throws IOException {
-    return compact(false);
-  }
-
-  /**
-   * Compacts the mob files by compaction type for the current column family.
-   * @param isForceAllFiles Whether add all mob files into the compaction.
-   * @return The paths of new mob files generated in the compaction.
-   * @throws IOException
-   */
-  public List<Path> compact(boolean isForceAllFiles) throws IOException {
-    return compact(Arrays.asList(fs.listStatus(mobFamilyDir)), isForceAllFiles);
-  }
-
-  /**
-   * Compacts the candidate mob files.
-   * @param files The candidate mob files.
-   * @param isForceAllFiles Whether add all mob files into the compaction.
-   * @return The paths of new mob files generated in the compaction.
-   * @throws IOException
-   */
-  public abstract List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
-    throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactionRequest.java
deleted file mode 100644
index d2ac1db..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactionRequest.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-/**
- * An implementation of {@link MobFileCompactionRequest} that is used in
- * {@link PartitionedMobFileCompactor}.
- * The mob files that have the same start key and date in their names belong to
- * the same partition.
- */
-@InterfaceAudience.Private
-public class PartitionedMobFileCompactionRequest extends MobFileCompactionRequest {
-
-  protected Collection<FileStatus> delFiles;
-  protected Collection<CompactionPartition> compactionPartitions;
-
-  public PartitionedMobFileCompactionRequest(Collection<CompactionPartition> compactionPartitions,
-    Collection<FileStatus> delFiles) {
-    this.selectionTime = EnvironmentEdgeManager.currentTime();
-    this.compactionPartitions = compactionPartitions;
-    this.delFiles = delFiles;
-  }
-
-  /**
-   * Gets the compaction partitions.
-   * @return The compaction partitions.
-   */
-  public Collection<CompactionPartition> getCompactionPartitions() {
-    return this.compactionPartitions;
-  }
-
-  /**
-   * Gets the del files.
-   * @return The del files.
-   */
-  public Collection<FileStatus> getDelFiles() {
-    return this.delFiles;
-  }
-
-  /**
-   * The partition in the mob file compaction.
-   * The mob files that have the same start key and date in their names belong to
-   * the same partition.
-   */
-  protected static class CompactionPartition {
-    private List<FileStatus> files = new ArrayList<FileStatus>();
-    private CompactionPartitionId partitionId;
-
-    public CompactionPartition(CompactionPartitionId partitionId) {
-      this.partitionId = partitionId;
-    }
-
-    public CompactionPartitionId getPartitionId() {
-      return this.partitionId;
-    }
-
-    public void addFile(FileStatus file) {
-      files.add(file);
-    }
-
-    public List<FileStatus> listFiles() {
-      return Collections.unmodifiableList(files);
-    }
-  }
-
-  /**
-   * The partition id that consists of start key and date of the mob file name.
-   */
-  protected static class CompactionPartitionId {
-
-    private String startKey;
-    private String date;
-
-    public CompactionPartitionId(String startKey, String date) {
-      if (startKey == null || date == null) {
-        throw new IllegalArgumentException("Neither of start key and date could be null");
-      }
-      this.startKey = startKey;
-      this.date = date;
-    }
-
-    public String getStartKey() {
-      return this.startKey;
-    }
-
-    public String getDate() {
-      return this.date;
-    }
-
-    @Override
-    public int hashCode() {
-      int result = 17;
-      result = 31 * result + startKey.hashCode();
-      result = 31 * result + date.hashCode();
-      return result;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      if (this == obj) {
-        return true;
-      }
-      if (!(obj instanceof CompactionPartitionId)) {
-        return false;
-      }
-      CompactionPartitionId another = (CompactionPartitionId) obj;
-      if (!this.startKey.equals(another.startKey)) {
-        return false;
-      }
-      if (!this.date.equals(another.date)) {
-        return false;
-      }
-      return true;
-    }
-
-    @Override
-    public String toString() {
-      return new StringBuilder(startKey).append(date).toString();
-    }
-  }
-}

[3/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
new file mode 100644
index 0000000..d63bb95
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -0,0 +1,924 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.security.Key;
+import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
+import org.apache.hadoop.hbase.io.crypto.aes.AES;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.mob.compactions.MobCompactor;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.security.EncryptionUtil;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestMobCompactor {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private Configuration conf = null;
+  private String tableNameAsString;
+  private TableName tableName;
+  private static Connection conn;
+  private BufferedMutator bufMut;
+  private Table hTable;
+  private Admin admin;
+  private HTableDescriptor desc;
+  private HColumnDescriptor hcd1;
+  private HColumnDescriptor hcd2;
+  private FileSystem fs;
+  private static final String family1 = "family1";
+  private static final String family2 = "family2";
+  private static final String qf1 = "qualifier1";
+  private static final String qf2 = "qualifier2";
+  private static byte[] KEYS = Bytes.toBytes("012");
+  private static int regionNum = KEYS.length;
+  private static int delRowNum = 1;
+  private static int delCellNum = 6;
+  private static int cellNumPerRow = 3;
+  private static int rowNumPerFile = 2;
+  private static ExecutorService pool;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+    TEST_UTIL.getConfiguration()
+      .setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, 5000);
+    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
+      KeyProviderForTesting.class.getName());
+    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
+    TEST_UTIL.startMiniCluster(1);
+    pool = createThreadPool(TEST_UTIL.getConfiguration());
+    conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    pool.shutdown();
+    conn.close();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    fs = TEST_UTIL.getTestFileSystem();
+    conf = TEST_UTIL.getConfiguration();
+    long tid = System.currentTimeMillis();
+    tableNameAsString = "testMob" + tid;
+    tableName = TableName.valueOf(tableNameAsString);
+    hcd1 = new HColumnDescriptor(family1);
+    hcd1.setMobEnabled(true);
+    hcd1.setMobThreshold(0L);
+    hcd1.setMaxVersions(4);
+    hcd2 = new HColumnDescriptor(family2);
+    hcd2.setMobEnabled(true);
+    hcd2.setMobThreshold(0L);
+    hcd2.setMaxVersions(4);
+    desc = new HTableDescriptor(tableName);
+    desc.addFamily(hcd1);
+    desc.addFamily(hcd2);
+    admin = TEST_UTIL.getHBaseAdmin();
+    admin.createTable(desc, getSplitKeys());
+    hTable = conn.getTable(tableName);
+    bufMut = conn.getBufferedMutator(tableName);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+    admin.close();
+    hTable.close();
+    fs.delete(TEST_UTIL.getDataTestDir(), true);
+  }
+
+  @Test
+  public void testCompactionWithoutDelFilesWithNamespace() throws Exception {
+    resetConf();
+    // create a table with namespace
+    NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
+    String tableNameAsString = "ns:testCompactionWithoutDelFilesWithNamespace";
+    admin.createNamespace(namespaceDescriptor);
+    TableName tableName = TableName.valueOf(tableNameAsString);
+    HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
+    hcd1.setMobEnabled(true);
+    hcd1.setMobThreshold(0L);
+    hcd1.setMaxVersions(4);
+    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
+    hcd2.setMobEnabled(true);
+    hcd2.setMobThreshold(0L);
+    hcd2.setMaxVersions(4);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(hcd1);
+    desc.addFamily(hcd2);
+    admin.createTable(desc, getSplitKeys());
+    BufferedMutator bufMut= conn.getBufferedMutator(tableName);
+    Table table = conn.getTable(tableName);
+
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count * rowNumPerFile;
+
+    assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
+      countMobRows(table));
+    assertEquals("Before compaction: mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
+
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
+      countMobRows(table));
+    assertEquals("After compaction: mob file count", regionNum,
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
+
+    table.close();
+    admin.disableTable(tableName);
+    admin.deleteTable(tableName);
+    admin.deleteNamespace("ns");
+  }
+
+  @Test
+  public void testCompactionWithoutDelFiles() throws Exception {
+    resetConf();
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
+
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("After compaction: mob file count", regionNum,
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
+  }
+
+  @Test
+  public void testCompactionWithoutDelFilesAndWithEncryption() throws Exception {
+    resetConf();
+    Configuration conf = TEST_UTIL.getConfiguration();
+    SecureRandom rng = new SecureRandom();
+    byte[] keyBytes = new byte[AES.KEY_LENGTH];
+    rng.nextBytes(keyBytes);
+    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
+    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
+    byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
+      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
+    String tableNameAsString = "testCompactionWithoutDelFilesAndWithEncryption";
+    TableName tableName = TableName.valueOf(tableNameAsString);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd = new HColumnDescriptor(family1);
+    hcd.setMobEnabled(true);
+    hcd.setMobThreshold(0);
+    hcd.setMaxVersions(4);
+    hcd.setEncryptionType(algorithm);
+    hcd.setEncryptionKey(encryptionKey);
+    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
+    hcd2.setMobEnabled(true);
+    hcd2.setMobThreshold(0);
+    hcd2.setMaxVersions(4);
+    desc.addFamily(hcd);
+    desc.addFamily(hcd2);
+    admin.createTable(desc, getSplitKeys());
+    Table hTable = conn.getTable(tableName);
+    BufferedMutator bufMut = conn.getBufferedMutator(tableName);
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
+
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("After compaction: mob file count", regionNum,
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
+    Assert.assertTrue(verifyEncryption(tableName, family1));
+  }
+
+  @Test
+  public void testCompactionWithDelFiles() throws Exception {
+    resetConf();
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
+        countMobCells(hTable));
+    assertEquals("Before deleting: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before deleting: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    // do the mob file compaction
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("After compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: family1 mob file count", regionNum,
+        countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After compaction: family1 del file count", 0,
+      countFiles(tableName, false, family1));
+    assertEquals("After compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+    assertRefFileNameEqual(family1);
+  }
+
+  @Test
+  public void testCompactionWithDelFilesAndNotMergeAllFiles() throws Exception {
+    resetConf();
+    int mergeSize = 5000;
+    // change the mob compaction merge size
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
+
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
+        countMobCells(hTable));
+    assertEquals("Before deleting: mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+
+    int largeFilesCount = countLargeFiles(mergeSize, family1);
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    // do the mob file compaction
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("After compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    // After the compaction, the files smaller than the mob compaction merge size
+    // is merge to one file
+    assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
+        countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("After compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+  }
+
+  @Test
+  public void testCompactionWithDelFilesAndWithSmallCompactionBatchSize() throws Exception {
+    resetConf();
+    int batchSize = 2;
+    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before deleting: mob row count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before deleting: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before deleting: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    // do the mob compaction
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("After compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
+        countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After compaction: family1 del file count", 0,
+      countFiles(tableName, false, family1));
+    assertEquals("After compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+  }
+
+  @Test
+  public void testCompactionWithHFileLink() throws IOException, InterruptedException {
+    resetConf();
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    long tid = System.currentTimeMillis();
+    byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
+    // take a snapshot
+    admin.snapshot(snapshotName1, tableName);
+
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    // do the mob compaction
+    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
+    compactor.compact();
+
+    assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("After first compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After first compaction: family1 mob file count", regionNum,
+        countFiles(tableName, true, family1));
+    assertEquals("After first compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After first compaction: family1 del file count", 0,
+      countFiles(tableName, false, family1));
+    assertEquals("After first compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+    assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
+    assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
+
+    admin.disableTable(tableName);
+    // Restore from snapshot, the hfilelink will exist in mob dir
+    admin.restoreSnapshot(snapshotName1);
+    admin.enableTable(tableName);
+
+    assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("After restoring snapshot: mob cells count",
+        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
+    assertEquals("After restoring snapshot: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("After restoring snapshot: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After restoring snapshot: family1 del file count", 0,
+        countFiles(tableName, false, family1));
+    assertEquals("After restoring snapshot: family2 del file count", 0,
+        countFiles(tableName, false, family2));
+    assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count,
+        countHFileLinks(family1));
+    assertEquals("After restoring snapshot: family2 hfilelink count", 0,
+        countHFileLinks(family2));
+
+    compactor.compact();
+
+    assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("After second compaction: mob cells count",
+        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
+    assertEquals("After second compaction: family1 mob file count", regionNum,
+        countFiles(tableName, true, family1));
+    assertEquals("After second compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After second compaction: family1 del file count", 0,
+      countFiles(tableName, false, family1));
+    assertEquals("After second compaction: family2 del file count", 0,
+      countFiles(tableName, false, family2));
+    assertEquals("After second compaction: family1 hfilelink count", 0, countHFileLinks(family1));
+    assertEquals("After second compaction: family2 hfilelink count", 0, countHFileLinks(family2));
+    assertRefFileNameEqual(family1);
+  }
+
+  @Test
+  public void testCompactionFromAdmin() throws Exception {
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
+        countMobCells(hTable));
+    assertEquals("Before deleting: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before deleting: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    int largeFilesCount = countLargeFiles(5000, family1);
+    // do the mob compaction
+    admin.compactMob(tableName, hcd1.getName());
+
+    waitUntilCompactionFinished(tableName);
+    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
+      countMobRows(hTable));
+    assertEquals("After compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount,
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
+    assertEquals("After compaction: family1 del file count", regionNum,
+      countFiles(tableName, false, family1));
+    assertEquals("After compaction: family2 del file count", regionNum,
+      countFiles(tableName, false, family2));
+    assertRefFileNameEqual(family1);
+  }
+
+  @Test
+  public void testMajorCompactionFromAdmin() throws Exception {
+    int count = 4;
+    // generate mob files
+    loadData(admin, bufMut, tableName, count, rowNumPerFile);
+    int rowNumPerRegion = count*rowNumPerFile;
+
+    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
+        countMobRows(hTable));
+    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
+        countMobCells(hTable));
+    assertEquals("Before deleting: mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+
+    createDelFile();
+
+    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("Before compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("Before compaction: family1 mob file count", regionNum*count,
+        countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family1 del file count", regionNum,
+        countFiles(tableName, false, family1));
+    assertEquals("Before compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+
+    // do the major mob compaction, it will force all files to compaction
+    admin.majorCompactMob(tableName, hcd1.getName());
+
+    waitUntilCompactionFinished(tableName);
+    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
+        countMobRows(hTable));
+    assertEquals("After compaction: mob cells count",
+        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: family1 mob file count", regionNum,
+        countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum*count,
+        countFiles(tableName, true, family2));
+    assertEquals("After compaction: family1 del file count", 0,
+        countFiles(tableName, false, family1));
+    assertEquals("After compaction: family2 del file count", regionNum,
+        countFiles(tableName, false, family2));
+  }
+
+  private void waitUntilCompactionFinished(TableName tableName) throws IOException,
+    InterruptedException {
+    long finished = EnvironmentEdgeManager.currentTime() + 60000;
+    CompactionState state = admin.getMobCompactionState(tableName);
+    while (EnvironmentEdgeManager.currentTime() < finished) {
+      if (state == CompactionState.NONE) {
+        break;
+      }
+      state = admin.getMobCompactionState(tableName);
+      Thread.sleep(10);
+    }
+    assertEquals(CompactionState.NONE, state);
+  }
+
+  /**
+   * Gets the number of rows in the given table.
+   * @param table to get the  scanner
+   * @return the number of rows
+   */
+  private int countMobRows(final Table table) throws IOException {
+    Scan scan = new Scan();
+    // Do not retrieve the mob data when scanning
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      count++;
+    }
+    results.close();
+    return count;
+  }
+
+  /**
+   * Gets the number of cells in the given table.
+   * @param table to get the  scanner
+   * @return the number of cells
+   */
+  private int countMobCells(final Table table) throws IOException {
+    Scan scan = new Scan();
+    // Do not retrieve the mob data when scanning
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      for (Cell cell : res.listCells()) {
+        count++;
+      }
+    }
+    results.close();
+    return count;
+  }
+
+  /**
+   * Gets the number of files in the mob path.
+   * @param isMobFile gets number of the mob files or del files
+   * @param familyName the family name
+   * @return the number of the files
+   */
+  private int countFiles(TableName tableName, boolean isMobFile, String familyName)
+    throws IOException {
+    Path mobDirPath = MobUtils.getMobFamilyPath(
+        MobUtils.getMobRegionPath(conf, tableName), familyName);
+    int count = 0;
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = fs.listStatus(mobDirPath);
+      for (FileStatus file : files) {
+        if (isMobFile == true) {
+          if (!StoreFileInfo.isDelFile(file.getPath())) {
+            count++;
+          }
+        } else {
+          if (StoreFileInfo.isDelFile(file.getPath())) {
+            count++;
+          }
+        }
+      }
+    }
+    return count;
+  }
+
+  private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
+    Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
+      familyName);
+    boolean hasFiles = false;
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = fs.listStatus(mobDirPath);
+      hasFiles = files != null && files.length > 0;
+      Assert.assertTrue(hasFiles);
+      Path path = files[0].getPath();
+      CacheConfig cacheConf = new CacheConfig(conf);
+      StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf,
+        BloomType.NONE);
+      HFile.Reader reader = sf.createReader().getHFileReader();
+      byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
+      Assert.assertTrue(null != encryptionKey);
+      Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
+        .equals(HConstants.CIPHER_AES));
+    }
+    return hasFiles;
+  }
+
+  /**
+   * Gets the number of HFileLink in the mob path.
+   * @param familyName the family name
+   * @return the number of the HFileLink
+   */
+  private int countHFileLinks(String familyName) throws IOException {
+    Path mobDirPath = MobUtils.getMobFamilyPath(
+        MobUtils.getMobRegionPath(conf, tableName), familyName);
+    int count = 0;
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = fs.listStatus(mobDirPath);
+      for (FileStatus file : files) {
+        if (HFileLink.isHFileLink(file.getPath())) {
+          count++;
+        }
+      }
+    }
+    return count;
+  }
+
+  /**
+   * Gets the number of files.
+   * @param size the size of the file
+   * @param familyName the family name
+   * @return the number of files large than the size
+   */
+  private int countLargeFiles(int size, String familyName) throws IOException {
+    Path mobDirPath = MobUtils.getMobFamilyPath(
+        MobUtils.getMobRegionPath(conf, tableName), familyName);
+    int count = 0;
+    if (fs.exists(mobDirPath)) {
+      FileStatus[] files = fs.listStatus(mobDirPath);
+      for (FileStatus file : files) {
+        // ignore the del files in the mob path
+        if ((!StoreFileInfo.isDelFile(file.getPath()))
+            && (file.getLen() > size)) {
+          count++;
+        }
+      }
+    }
+    return count;
+  }
+
+  /**
+   * loads some data to the table.
+   */
+  private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum,
+    int rowNumPerFile) throws IOException, InterruptedException {
+    if (fileNum <= 0) {
+      throw new IllegalArgumentException();
+    }
+    for (byte k0 : KEYS) {
+      byte[] k = new byte[] { k0 };
+      for (int i = 0; i < fileNum * rowNumPerFile; i++) {
+        byte[] key = Bytes.add(k, Bytes.toBytes(i));
+        byte[] mobVal = makeDummyData(10 * (i + 1));
+        Put put = new Put(key);
+        put.setDurability(Durability.SKIP_WAL);
+        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal);
+        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
+        put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
+        table.mutate(put);
+        if ((i + 1) % rowNumPerFile == 0) {
+          table.flush();
+          admin.flush(tableName);
+        }
+      }
+    }
+  }
+
+  /**
+   * delete the row, family and cell to create the del file
+   */
+  private void createDelFile() throws IOException, InterruptedException {
+    for (byte k0 : KEYS) {
+      byte[] k = new byte[] { k0 };
+      // delete a family
+      byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
+      Delete delete1 = new Delete(key1);
+      delete1.addFamily(Bytes.toBytes(family1));
+      hTable.delete(delete1);
+      // delete one row
+      byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
+      Delete delete2 = new Delete(key2);
+      hTable.delete(delete2);
+      // delete one cell
+      byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
+      Delete delete3 = new Delete(key3);
+      delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1));
+      hTable.delete(delete3);
+      admin.flush(tableName);
+      List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(
+          Bytes.toBytes(tableNameAsString));
+      for (HRegion region : regions) {
+        region.waitForFlushesAndCompactions();
+        region.compact(true);
+      }
+    }
+  }
+  /**
+   * Creates the dummy data with a specific size.
+   * @param size the size of value
+   * @return the dummy data
+   */
+  private byte[] makeDummyData(int size) {
+    byte[] dummyData = new byte[size];
+    new Random().nextBytes(dummyData);
+    return dummyData;
+  }
+
+  /**
+   * Gets the split keys
+   */
+  private byte[][] getSplitKeys() {
+    byte[][] splitKeys = new byte[KEYS.length - 1][];
+    for (int i = 0; i < splitKeys.length; ++i) {
+      splitKeys[i] = new byte[] { KEYS[i + 1] };
+    }
+    return splitKeys;
+  }
+
+  private static ExecutorService createThreadPool(Configuration conf) {
+    int maxThreads = 10;
+    long keepAliveTime = 60;
+    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
+    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
+        keepAliveTime, TimeUnit.SECONDS, queue,
+        Threads.newDaemonThreadFactory("MobFileCompactionChore"),
+        new RejectedExecutionHandler() {
+          @Override
+          public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+            try {
+              // waiting for a thread to pick up instead of throwing exceptions.
+              queue.put(r);
+            } catch (InterruptedException e) {
+              throw new RejectedExecutionException(e);
+            }
+          }
+        });
+    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
+    return pool;
+  }
+
+  private void assertRefFileNameEqual(String familyName) throws IOException {
+    Scan scan = new Scan();
+    scan.addFamily(Bytes.toBytes(familyName));
+    // Do not retrieve the mob data when scanning
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner results = hTable.getScanner(scan);
+    Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
+        tableName), familyName);
+    List<Path> actualFilePaths = new ArrayList<>();
+    List<Path> expectFilePaths = new ArrayList<>();
+    for (Result res : results) {
+      for (Cell cell : res.listCells()) {
+        byte[] referenceValue = CellUtil.cloneValue(cell);
+        String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
+            referenceValue.length - Bytes.SIZEOF_INT);
+        Path targetPath = new Path(mobFamilyPath, fileName);
+        if(!actualFilePaths.contains(targetPath)) {
+          actualFilePaths.add(targetPath);
+        }
+      }
+    }
+    results.close();
+    if (fs.exists(mobFamilyPath)) {
+      FileStatus[] files = fs.listStatus(mobFamilyPath);
+      for (FileStatus file : files) {
+        if (!StoreFileInfo.isDelFile(file.getPath())) {
+          expectFilePaths.add(file.getPath());
+        }
+      }
+    }
+    Collections.sort(actualFilePaths);
+    Collections.sort(expectFilePaths);
+    assertEquals(expectFilePaths, actualFilePaths);
+  }
+
+  /**
+   * Resets the configuration.
+   */
+  private void resetConf() {
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
+      MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
+    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
+      MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactionRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactionRequest.java
new file mode 100644
index 0000000..fabc4e2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactionRequest.java
@@ -0,0 +1,60 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartitionId;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestPartitionedMobCompactionRequest {
+
+  @Test
+  public void testCompactedPartitionId() {
+    String startKey1 = "startKey1";
+    String startKey2 = "startKey2";
+    String date1 = "date1";
+    String date2 = "date2";
+    CompactionPartitionId partitionId1 = new CompactionPartitionId(startKey1, date1);
+    CompactionPartitionId partitionId2 = new CompactionPartitionId(startKey2, date2);
+    CompactionPartitionId partitionId3 = new CompactionPartitionId(startKey1, date2);
+
+    Assert.assertTrue(partitionId1.equals(partitionId1));
+    Assert.assertFalse(partitionId1.equals(partitionId2));
+    Assert.assertFalse(partitionId1.equals(partitionId3));
+    Assert.assertFalse(partitionId2.equals(partitionId3));
+
+    Assert.assertEquals(startKey1, partitionId1.getStartKey());
+    Assert.assertEquals(date1, partitionId1.getDate());
+  }
+
+  @Test
+  public void testCompactedPartition() {
+    CompactionPartitionId partitionId = new CompactionPartitionId("startKey1", "date1");
+    CompactionPartition partition = new CompactionPartition(partitionId);
+    FileStatus file = new FileStatus(1, false, 1, 1024, 1, new Path("/test"));
+    partition.addFile(file);
+    Assert.assertEquals(file, partition.listFiles().get(0));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
new file mode 100644
index 0000000..e6d2b98
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
@@ -0,0 +1,440 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob.compactions;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobFileName;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
+import org.apache.hadoop.hbase.mob.compactions.MobCompactionRequest.CompactionType;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartition;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestPartitionedMobCompactor {
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static String family = "family";
+  private final static String qf = "qf";
+  private HColumnDescriptor hcd = new HColumnDescriptor(family);
+  private Configuration conf = TEST_UTIL.getConfiguration();
+  private CacheConfig cacheConf = new CacheConfig(conf);
+  private FileSystem fs;
+  private List<FileStatus> mobFiles = new ArrayList<>();
+  private List<FileStatus> delFiles = new ArrayList<>();
+  private List<FileStatus> allFiles = new ArrayList<>();
+  private Path basePath;
+  private String mobSuffix;
+  private String delSuffix;
+  private static ExecutorService pool;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+    TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
+    TEST_UTIL.startMiniCluster(1);
+    pool = createThreadPool();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    pool.shutdown();
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  private void init(String tableName) throws Exception {
+    fs = FileSystem.get(conf);
+    Path testDir = FSUtils.getRootDir(conf);
+    Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
+    basePath = new Path(new Path(mobTestDir, tableName), family);
+    mobSuffix = UUID.randomUUID().toString().replaceAll("-", "");
+    delSuffix = UUID.randomUUID().toString().replaceAll("-", "") + "_del";
+  }
+
+  @Test
+  public void testCompactionSelectWithAllFiles() throws Exception {
+    resetConf();
+    String tableName = "testCompactionSelectWithAllFiles";
+    init(tableName);
+    int count = 10;
+    // create 10 mob files.
+    createStoreFiles(basePath, family, qf, count, Type.Put);
+    // create 10 del files
+    createStoreFiles(basePath, family, qf, count, Type.Delete);
+    listFiles();
+    long mergeSize = MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD;
+    List<String> expectedStartKeys = new ArrayList<>();
+    for(FileStatus file : mobFiles) {
+      if(file.getLen() < mergeSize) {
+        String fileName = file.getPath().getName();
+        String startKey = fileName.substring(0, 32);
+        expectedStartKeys.add(startKey);
+      }
+    }
+    testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
+  }
+
+  @Test
+  public void testCompactionSelectWithPartFiles() throws Exception {
+    resetConf();
+    String tableName = "testCompactionSelectWithPartFiles";
+    init(tableName);
+    int count = 10;
+    // create 10 mob files.
+    createStoreFiles(basePath, family, qf, count, Type.Put);
+    // create 10 del files
+    createStoreFiles(basePath, family, qf, count, Type.Delete);
+    listFiles();
+    long mergeSize = 4000;
+    List<String> expectedStartKeys = new ArrayList<>();
+    for(FileStatus file : mobFiles) {
+      if(file.getLen() < 4000) {
+        String fileName = file.getPath().getName();
+        String startKey = fileName.substring(0, 32);
+        expectedStartKeys.add(startKey);
+      }
+    }
+    // set the mob compaction mergeable threshold
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
+    testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
+  }
+
+  @Test
+  public void testCompactionSelectWithForceAllFiles() throws Exception {
+    resetConf();
+    String tableName = "testCompactionSelectWithForceAllFiles";
+    init(tableName);
+    int count = 10;
+    // create 10 mob files.
+    createStoreFiles(basePath, family, qf, count, Type.Put);
+    // create 10 del files
+    createStoreFiles(basePath, family, qf, count, Type.Delete);
+    listFiles();
+    long mergeSize = 4000;
+    List<String> expectedStartKeys = new ArrayList<>();
+    for(FileStatus file : mobFiles) {
+      String fileName = file.getPath().getName();
+      String startKey = fileName.substring(0, 32);
+      expectedStartKeys.add(startKey);
+    }
+    // set the mob compaction mergeable threshold
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
+    testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
+  }
+
+  @Test
+  public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
+    resetConf();
+    String tableName = "testCompactDelFilesWithDefaultBatchSize";
+    init(tableName);
+    // create 20 mob files.
+    createStoreFiles(basePath, family, qf, 20, Type.Put);
+    // create 13 del files
+    createStoreFiles(basePath, family, qf, 13, Type.Delete);
+    listFiles();
+    testCompactDelFiles(tableName, 1, 13, false);
+  }
+
+  @Test
+  public void testCompactDelFilesWithSmallBatchSize() throws Exception {
+    resetConf();
+    String tableName = "testCompactDelFilesWithSmallBatchSize";
+    init(tableName);
+    // create 20 mob files.
+    createStoreFiles(basePath, family, qf, 20, Type.Put);
+    // create 13 del files
+    createStoreFiles(basePath, family, qf, 13, Type.Delete);
+    listFiles();
+
+    // set the mob compaction batch size
+    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 4);
+    testCompactDelFiles(tableName, 1, 13, false);
+  }
+
+  @Test
+  public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
+    resetConf();
+    String tableName = "testCompactDelFilesWithSmallBatchSize";
+    init(tableName);
+    // create 20 mob files.
+    createStoreFiles(basePath, family, qf, 20, Type.Put);
+    // create 13 del files
+    createStoreFiles(basePath, family, qf, 13, Type.Delete);
+    listFiles();
+
+    // set the max del file count
+    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
+    // set the mob compaction batch size
+    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, 2);
+    testCompactDelFiles(tableName, 4, 13, false);
+  }
+
+  /**
+   * Tests the selectFiles
+   * @param tableName the table name
+   * @param type the expected compaction type
+   * @param isForceAllFiles whether all the mob files are selected
+   * @param expected the expected start keys
+   */
+  private void testSelectFiles(String tableName, final CompactionType type,
+    final boolean isForceAllFiles, final List<String> expected) throws IOException {
+    PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
+      TableName.valueOf(tableName), hcd, pool) {
+      @Override
+      public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
+        throws IOException {
+        if (files == null || files.isEmpty()) {
+          return null;
+        }
+        PartitionedMobCompactionRequest request = select(files, isForceAllFiles);
+        // assert the compaction type
+        Assert.assertEquals(type, request.type);
+        // assert get the right partitions
+        compareCompactedPartitions(expected, request.compactionPartitions);
+        // assert get the right del files
+        compareDelFiles(request.delFiles);
+        return null;
+      }
+    };
+    compactor.compact(allFiles, isForceAllFiles);
+  }
+
+  /**
+   * Tests the compacteDelFile
+   * @param tableName the table name
+   * @param expectedFileCount the expected file count
+   * @param expectedCellCount the expected cell count
+   * @param isForceAllFiles whether all the mob files are selected
+   */
+  private void testCompactDelFiles(String tableName, final int expectedFileCount,
+      final int expectedCellCount, boolean isForceAllFiles) throws IOException {
+    PartitionedMobCompactor compactor = new PartitionedMobCompactor(conf, fs,
+      TableName.valueOf(tableName), hcd, pool) {
+      @Override
+      protected List<Path> performCompaction(PartitionedMobCompactionRequest request)
+          throws IOException {
+        List<Path> delFilePaths = new ArrayList<Path>();
+        for (FileStatus delFile : request.delFiles) {
+          delFilePaths.add(delFile.getPath());
+        }
+        List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
+        // assert the del files are merged.
+        Assert.assertEquals(expectedFileCount, newDelPaths.size());
+        Assert.assertEquals(expectedCellCount, countDelCellsInDelFiles(newDelPaths));
+        return null;
+      }
+    };
+    compactor.compact(allFiles, isForceAllFiles);
+  }
+
+  /**
+   * Lists the files in the path
+   */
+  private void listFiles() throws IOException {
+    for (FileStatus file : fs.listStatus(basePath)) {
+      allFiles.add(file);
+      if (file.getPath().getName().endsWith("_del")) {
+        delFiles.add(file);
+      } else {
+        mobFiles.add(file);
+      }
+    }
+  }
+
+  /**
+   * Compares the compacted partitions.
+   * @param partitions the collection of CompactedPartitions
+   */
+  private void compareCompactedPartitions(List<String> expected,
+      Collection<CompactionPartition> partitions) {
+    List<String> actualKeys = new ArrayList<>();
+    for (CompactionPartition partition : partitions) {
+      actualKeys.add(partition.getPartitionId().getStartKey());
+    }
+    Collections.sort(expected);
+    Collections.sort(actualKeys);
+    Assert.assertEquals(expected.size(), actualKeys.size());
+    for (int i = 0; i < expected.size(); i++) {
+      Assert.assertEquals(expected.get(i), actualKeys.get(i));
+    }
+  }
+
+  /**
+   * Compares the del files.
+   * @param allDelFiles all the del files
+   */
+  private void compareDelFiles(Collection<FileStatus> allDelFiles) {
+    int i = 0;
+    for (FileStatus file : allDelFiles) {
+      Assert.assertEquals(delFiles.get(i), file);
+      i++;
+    }
+  }
+
+  /**
+   * Creates store files.
+   * @param basePath the path to create file
+   * @family the family name
+   * @qualifier the column qualifier
+   * @count the store file number
+   * @type the key type
+   */
+  private void createStoreFiles(Path basePath, String family, String qualifier, int count,
+      Type type) throws IOException {
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
+    String startKey = "row_";
+    MobFileName mobFileName = null;
+    for (int i = 0; i < count; i++) {
+      byte[] startRow = Bytes.toBytes(startKey + i) ;
+      if(type.equals(Type.Delete)) {
+        mobFileName = MobFileName.create(startRow, MobUtils.formatDate(
+            new Date()), delSuffix);
+      }
+      if(type.equals(Type.Put)){
+        mobFileName = MobFileName.create(Bytes.toBytes(startKey + i), MobUtils.formatDate(
+            new Date()), mobSuffix);
+      }
+      StoreFile.Writer mobFileWriter = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+      .withFileContext(meta).withFilePath(new Path(basePath, mobFileName.getFileName())).build();
+      writeStoreFile(mobFileWriter, startRow, Bytes.toBytes(family), Bytes.toBytes(qualifier),
+          type, (i+1)*1000);
+    }
+  }
+
+  /**
+   * Writes data to store file.
+   * @param writer the store file writer
+   * @param row the row key
+   * @param family the family name
+   * @param qualifier the column qualifier
+   * @param type the key type
+   * @param size the size of value
+   */
+  private static void writeStoreFile(final StoreFile.Writer writer, byte[]row, byte[] family,
+      byte[] qualifier, Type type, int size) throws IOException {
+    long now = System.currentTimeMillis();
+    try {
+      byte[] dummyData = new byte[size];
+      new Random().nextBytes(dummyData);
+      writer.append(new KeyValue(row, family, qualifier, now, type, dummyData));
+    } finally {
+      writer.close();
+    }
+  }
+
+  /**
+   * Gets the number of del cell in the del files
+   * @param paths the del file paths
+   * @return the cell size
+   */
+  private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
+    List<StoreFile> sfs = new ArrayList<StoreFile>();
+    int size = 0;
+    for(Path path : paths) {
+      StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.NONE);
+      sfs.add(sf);
+    }
+    List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
+        false, null, HConstants.LATEST_TIMESTAMP);
+    Scan scan = new Scan();
+    scan.setMaxVersions(hcd.getMaxVersions());
+    long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
+    long ttl = HStore.determineTTLFromFamily(hcd);
+    ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
+    StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
+        scanners, 0L, HConstants.LATEST_TIMESTAMP);
+    List<Cell> results = new ArrayList<>();
+    boolean hasMore = true;
+
+    while (hasMore) {
+      hasMore = scanner.next(results);
+      size += results.size();
+      results.clear();
+    }
+    scanner.close();
+    return size;
+  }
+
+  private static ExecutorService createThreadPool() {
+    int maxThreads = 10;
+    long keepAliveTime = 60;
+    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
+    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
+      TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"),
+      new RejectedExecutionHandler() {
+        @Override
+        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+          try {
+            // waiting for a thread to pick up instead of throwing exceptions.
+            queue.put(r);
+          } catch (InterruptedException e) {
+            throw new RejectedExecutionException(e);
+          }
+        }
+      });
+    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
+    return pool;
+  }
+
+  /**
+   * Resets the configuration.
+   */
+  private void resetConf() {
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD,
+      MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD);
+    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
+    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE,
+      MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE);
+  }
+}
\ No newline at end of file

[2/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
deleted file mode 100644
index abdfb94..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
+++ /dev/null
@@ -1,922 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.security.Key;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
-import org.apache.hadoop.hbase.io.crypto.aes.AES;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.security.EncryptionUtil;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Threads;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestMobFileCompactor {
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private Configuration conf = null;
-  private String tableNameAsString;
-  private TableName tableName;
-  private static Connection conn;
-  private BufferedMutator bufMut;
-  private Table hTable;
-  private Admin admin;
-  private HTableDescriptor desc;
-  private HColumnDescriptor hcd1;
-  private HColumnDescriptor hcd2;
-  private FileSystem fs;
-  private static final String family1 = "family1";
-  private static final String family2 = "family2";
-  private static final String qf1 = "qualifier1";
-  private static final String qf2 = "qualifier2";
-  private static byte[] KEYS = Bytes.toBytes("012");
-  private static int regionNum = KEYS.length;
-  private static int delRowNum = 1;
-  private static int delCellNum = 6;
-  private static int cellNumPerRow = 3;
-  private static int rowNumPerFile = 2;
-  private static ExecutorService pool;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    TEST_UTIL.getConfiguration()
-      .setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, 5000);
-    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
-      KeyProviderForTesting.class.getName());
-    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
-    TEST_UTIL.startMiniCluster(1);
-    pool = createThreadPool(TEST_UTIL.getConfiguration());
-    conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    pool.shutdown();
-    conn.close();
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    fs = TEST_UTIL.getTestFileSystem();
-    conf = TEST_UTIL.getConfiguration();
-    long tid = System.currentTimeMillis();
-    tableNameAsString = "testMob" + tid;
-    tableName = TableName.valueOf(tableNameAsString);
-    hcd1 = new HColumnDescriptor(family1);
-    hcd1.setMobEnabled(true);
-    hcd1.setMobThreshold(0L);
-    hcd1.setMaxVersions(4);
-    hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0L);
-    hcd2.setMaxVersions(4);
-    desc = new HTableDescriptor(tableName);
-    desc.addFamily(hcd1);
-    desc.addFamily(hcd2);
-    admin = TEST_UTIL.getHBaseAdmin();
-    admin.createTable(desc, getSplitKeys());
-    hTable = conn.getTable(tableName);
-    bufMut = conn.getBufferedMutator(tableName);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.close();
-    hTable.close();
-    fs.delete(TEST_UTIL.getDataTestDir(), true);
-  }
-
-  @Test
-  public void testCompactionWithoutDelFilesWithNamespace() throws Exception {
-    resetConf();
-    // create a table with namespace
-    NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
-    String tableNameAsString = "ns:testCompactionWithoutDelFilesWithNamespace";
-    admin.createNamespace(namespaceDescriptor);
-    TableName tableName = TableName.valueOf(tableNameAsString);
-    HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
-    hcd1.setMobEnabled(true);
-    hcd1.setMobThreshold(0L);
-    hcd1.setMaxVersions(4);
-    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0L);
-    hcd2.setMaxVersions(4);
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    desc.addFamily(hcd1);
-    desc.addFamily(hcd2);
-    admin.createTable(desc, getSplitKeys());
-    BufferedMutator bufMut= conn.getBufferedMutator(tableName);
-    Table table = conn.getTable(tableName);
-
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count * rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
-      countMobRows(table));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
-      countMobRows(table));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    table.close();
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.deleteNamespace("ns");
-  }
-
-  @Test
-  public void testCompactionWithoutDelFiles() throws Exception {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-  }
-
-  @Test
-  public void testCompactionWithoutDelFilesAndWithEncryption() throws Exception {
-    resetConf();
-    Configuration conf = TEST_UTIL.getConfiguration();
-    SecureRandom rng = new SecureRandom();
-    byte[] keyBytes = new byte[AES.KEY_LENGTH];
-    rng.nextBytes(keyBytes);
-    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
-    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
-    byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
-      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
-    String tableNameAsString = "testCompactionWithoutDelFilesAndWithEncryption";
-    TableName tableName = TableName.valueOf(tableNameAsString);
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor(family1);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(0);
-    hcd.setMaxVersions(4);
-    hcd.setEncryptionType(algorithm);
-    hcd.setEncryptionKey(encryptionKey);
-    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0);
-    hcd2.setMaxVersions(4);
-    desc.addFamily(hcd);
-    desc.addFamily(hcd2);
-    admin.createTable(desc, getSplitKeys());
-    Table hTable = conn.getTable(tableName);
-    BufferedMutator bufMut = conn.getBufferedMutator(tableName);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-    Assert.assertTrue(verifyEncryption(tableName, family1));
-  }
-
-  @Test
-  public void testCompactionWithDelFiles() throws Exception {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testCompactionWithDelFilesAndNotMergeAllFiles() throws Exception {
-    resetConf();
-    int mergeSize = 5000;
-    // change the mob compaction merge size
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-
-    int largeFilesCount = countLargeFiles(mergeSize, family1);
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    // After the compaction, the files smaller than the mob compaction merge size
-    // is merge to one file
-    assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  @Test
-  public void testCompactionWithDelFilesAndWithSmallCompactionBatchSize() throws Exception {
-    resetConf();
-    int batchSize = 2;
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, batchSize);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob row count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  @Test
-  public void testCompactionWithHFileLink() throws IOException, InterruptedException {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    long tid = System.currentTimeMillis();
-    byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
-    // take a snapshot
-    admin.snapshot(snapshotName1, tableName);
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After first compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After first compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After first compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After first compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After first compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-    assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
-    assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
-
-    admin.disableTable(tableName);
-    // Restore from snapshot, the hfilelink will exist in mob dir
-    admin.restoreSnapshot(snapshotName1);
-    admin.enableTable(tableName);
-
-    assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After restoring snapshot: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
-    assertEquals("After restoring snapshot: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("After restoring snapshot: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After restoring snapshot: family1 del file count", 0,
-        countFiles(tableName, false, family1));
-    assertEquals("After restoring snapshot: family2 del file count", 0,
-        countFiles(tableName, false, family2));
-    assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count,
-        countHFileLinks(family1));
-    assertEquals("After restoring snapshot: family2 hfilelink count", 0,
-        countHFileLinks(family2));
-
-    compactor.compact();
-
-    assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After second compaction: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
-    assertEquals("After second compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After second compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After second compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After second compaction: family2 del file count", 0,
-      countFiles(tableName, false, family2));
-    assertEquals("After second compaction: family1 hfilelink count", 0, countHFileLinks(family1));
-    assertEquals("After second compaction: family2 hfilelink count", 0, countHFileLinks(family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testCompactionFromAdmin() throws Exception {
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    int largeFilesCount = countLargeFiles(5000, family1);
-    // do the mob file compaction
-    admin.compactMob(tableName, hcd1.getName());
-
-    waitUntilCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
-      countMobRows(hTable));
-    assertEquals("After compaction: mob cells count", regionNum
-      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum * count,
-      countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", regionNum,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-      countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testMajorCompactionFromAdmin() throws Exception {
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the major mob file compaction, it will force all files to compaction
-    admin.majorCompactMob(tableName, hcd1.getName());
-
-    waitUntilCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-        countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  private void waitUntilCompactionFinished(TableName tableName) throws IOException,
-    InterruptedException {
-    long finished = EnvironmentEdgeManager.currentTime() + 60000;
-    CompactionState state = admin.getMobCompactionState(tableName);
-    while (EnvironmentEdgeManager.currentTime() < finished) {
-      if (state == CompactionState.NONE) {
-        break;
-      }
-      state = admin.getMobCompactionState(tableName);
-      Thread.sleep(10);
-    }
-    assertEquals(CompactionState.NONE, state);
-  }
-
-  /**
-   * Gets the number of rows in the given table.
-   * @param table to get the  scanner
-   * @return the number of rows
-   */
-  private int countMobRows(final Table table) throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = table.getScanner(scan);
-    int count = 0;
-    for (Result res : results) {
-      count++;
-    }
-    results.close();
-    return count;
-  }
-
-  /**
-   * Gets the number of cells in the given table.
-   * @param table to get the  scanner
-   * @return the number of cells
-   */
-  private int countMobCells(final Table table) throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = table.getScanner(scan);
-    int count = 0;
-    for (Result res : results) {
-      for (Cell cell : res.listCells()) {
-        count++;
-      }
-    }
-    results.close();
-    return count;
-  }
-
-  /**
-   * Gets the number of files in the mob path.
-   * @param isMobFile gets number of the mob files or del files
-   * @param familyName the family name
-   * @return the number of the files
-   */
-  private int countFiles(TableName tableName, boolean isMobFile, String familyName)
-    throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        if (isMobFile == true) {
-          if (!StoreFileInfo.isDelFile(file.getPath())) {
-            count++;
-          }
-        } else {
-          if (StoreFileInfo.isDelFile(file.getPath())) {
-            count++;
-          }
-        }
-      }
-    }
-    return count;
-  }
-
-  private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
-      familyName);
-    boolean hasFiles = false;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      hasFiles = files != null && files.length > 0;
-      Assert.assertTrue(hasFiles);
-      Path path = files[0].getPath();
-      CacheConfig cacheConf = new CacheConfig(conf);
-      StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf,
-        BloomType.NONE);
-      HFile.Reader reader = sf.createReader().getHFileReader();
-      byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
-      Assert.assertTrue(null != encryptionKey);
-      Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
-        .equals(HConstants.CIPHER_AES));
-    }
-    return hasFiles;
-  }
-
-  /**
-   * Gets the number of HFileLink in the mob path.
-   * @param familyName the family name
-   * @return the number of the HFileLink
-   */
-  private int countHFileLinks(String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        if (HFileLink.isHFileLink(file.getPath())) {
-          count++;
-        }
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Gets the number of files.
-   * @param size the size of the file
-   * @param familyName the family name
-   * @return the number of files large than the size
-   */
-  private int countLargeFiles(int size, String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        // ignore the del files in the mob path
-        if ((!StoreFileInfo.isDelFile(file.getPath()))
-            && (file.getLen() > size)) {
-          count++;
-        }
-      }
-    }
-    return count;
-  }
-
-  /**
-   * loads some data to the table.
-   */
-  private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum,
-    int rowNumPerFile) throws IOException, InterruptedException {
-    if (fileNum <= 0) {
-      throw new IllegalArgumentException();
-    }
-    for (byte k0 : KEYS) {
-      byte[] k = new byte[] { k0 };
-      for (int i = 0; i < fileNum * rowNumPerFile; i++) {
-        byte[] key = Bytes.add(k, Bytes.toBytes(i));
-        byte[] mobVal = makeDummyData(10 * (i + 1));
-        Put put = new Put(key);
-        put.setDurability(Durability.SKIP_WAL);
-        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal);
-        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
-        put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
-        table.mutate(put);
-        if ((i + 1) % rowNumPerFile == 0) {
-          table.flush();
-          admin.flush(tableName);
-        }
-      }
-    }
-  }
-
-  /**
-   * delete the row, family and cell to create the del file
-   */
-  private void createDelFile() throws IOException, InterruptedException {
-    for (byte k0 : KEYS) {
-      byte[] k = new byte[] { k0 };
-      // delete a family
-      byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
-      Delete delete1 = new Delete(key1);
-      delete1.addFamily(Bytes.toBytes(family1));
-      hTable.delete(delete1);
-      // delete one row
-      byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
-      Delete delete2 = new Delete(key2);
-      hTable.delete(delete2);
-      // delete one cell
-      byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
-      Delete delete3 = new Delete(key3);
-      delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1));
-      hTable.delete(delete3);
-      admin.flush(tableName);
-      List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(
-          Bytes.toBytes(tableNameAsString));
-      for (HRegion region : regions) {
-        region.waitForFlushesAndCompactions();
-        region.compact(true);
-      }
-    }
-  }
-  /**
-   * Creates the dummy data with a specific size.
-   * @param size the size of value
-   * @return the dummy data
-   */
-  private byte[] makeDummyData(int size) {
-    byte[] dummyData = new byte[size];
-    new Random().nextBytes(dummyData);
-    return dummyData;
-  }
-
-  /**
-   * Gets the split keys
-   */
-  private byte[][] getSplitKeys() {
-    byte[][] splitKeys = new byte[KEYS.length - 1][];
-    for (int i = 0; i < splitKeys.length; ++i) {
-      splitKeys[i] = new byte[] { KEYS[i + 1] };
-    }
-    return splitKeys;
-  }
-
-  private static ExecutorService createThreadPool(Configuration conf) {
-    int maxThreads = 10;
-    long keepAliveTime = 60;
-    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
-    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
-        keepAliveTime, TimeUnit.SECONDS, queue,
-        Threads.newDaemonThreadFactory("MobFileCompactionChore"),
-        new RejectedExecutionHandler() {
-          @Override
-          public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
-            try {
-              // waiting for a thread to pick up instead of throwing exceptions.
-              queue.put(r);
-            } catch (InterruptedException e) {
-              throw new RejectedExecutionException(e);
-            }
-          }
-        });
-    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
-    return pool;
-  }
-
-  private void assertRefFileNameEqual(String familyName) throws IOException {
-    Scan scan = new Scan();
-    scan.addFamily(Bytes.toBytes(familyName));
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = hTable.getScanner(scan);
-    Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
-        tableName), familyName);
-    List<Path> actualFilePaths = new ArrayList<>();
-    List<Path> expectFilePaths = new ArrayList<>();
-    for (Result res : results) {
-      for (Cell cell : res.listCells()) {
-        byte[] referenceValue = CellUtil.cloneValue(cell);
-        String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
-            referenceValue.length - Bytes.SIZEOF_INT);
-        Path targetPath = new Path(mobFamilyPath, fileName);
-        if(!actualFilePaths.contains(targetPath)) {
-          actualFilePaths.add(targetPath);
-        }
-      }
-    }
-    results.close();
-    if (fs.exists(mobFamilyPath)) {
-      FileStatus[] files = fs.listStatus(mobFamilyPath);
-      for (FileStatus file : files) {
-        if (!StoreFileInfo.isDelFile(file.getPath())) {
-          expectFilePaths.add(file.getPath());
-        }
-      }
-    }
-    Collections.sort(actualFilePaths);
-    Collections.sort(expectFilePaths);
-    assertEquals(expectFilePaths, actualFilePaths);
-  }
-
-  /**
-   * Resets the configuration.
-   */
-  private void resetConf() {
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD);
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
deleted file mode 100644
index f9159aa..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartition;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartitionId;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(SmallTests.class)
-public class TestPartitionedMobFileCompactionRequest {
-
-  @Test
-  public void testCompactedPartitionId() {
-    String startKey1 = "startKey1";
-    String startKey2 = "startKey2";
-    String date1 = "date1";
-    String date2 = "date2";
-    CompactionPartitionId partitionId1 = new CompactionPartitionId(startKey1, date1);
-    CompactionPartitionId partitionId2 = new CompactionPartitionId(startKey2, date2);
-    CompactionPartitionId partitionId3 = new CompactionPartitionId(startKey1, date2);
-
-    Assert.assertTrue(partitionId1.equals(partitionId1));
-    Assert.assertFalse(partitionId1.equals(partitionId2));
-    Assert.assertFalse(partitionId1.equals(partitionId3));
-    Assert.assertFalse(partitionId2.equals(partitionId3));
-
-    Assert.assertEquals(startKey1, partitionId1.getStartKey());
-    Assert.assertEquals(date1, partitionId1.getDate());
-  }
-
-  @Test
-  public void testCompactedPartition() {
-    CompactionPartitionId partitionId = new CompactionPartitionId("startKey1", "date1");
-    CompactionPartition partition = new CompactionPartition(partitionId);
-    FileStatus file = new FileStatus(1, false, 1, 1024, 1, new Path("/test"));
-    partition.addFile(file);
-    Assert.assertEquals(file, partition.listFiles().get(0));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
deleted file mode 100644
index 544d145..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
-import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobFileName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactionRequest.CompactionType;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartition;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestPartitionedMobFileCompactor {
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final static String family = "family";
-  private final static String qf = "qf";
-  private HColumnDescriptor hcd = new HColumnDescriptor(family);
-  private Configuration conf = TEST_UTIL.getConfiguration();
-  private CacheConfig cacheConf = new CacheConfig(conf);
-  private FileSystem fs;
-  private List<FileStatus> mobFiles = new ArrayList<>();
-  private List<FileStatus> delFiles = new ArrayList<>();
-  private List<FileStatus> allFiles = new ArrayList<>();
-  private Path basePath;
-  private String mobSuffix;
-  private String delSuffix;
-  private static ExecutorService pool;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
-    TEST_UTIL.startMiniCluster(1);
-    pool = createThreadPool();
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    pool.shutdown();
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  private void init(String tableName) throws Exception {
-    fs = FileSystem.get(conf);
-    Path testDir = FSUtils.getRootDir(conf);
-    Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
-    basePath = new Path(new Path(mobTestDir, tableName), family);
-    mobSuffix = UUID.randomUUID().toString().replaceAll("-", "");
-    delSuffix = UUID.randomUUID().toString().replaceAll("-", "") + "_del";
-  }
-
-  @Test
-  public void testCompactionSelectWithAllFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithAllFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      if(file.getLen() < mergeSize) {
-        String fileName = file.getPath().getName();
-        String startKey = fileName.substring(0, 32);
-        expectedStartKeys.add(startKey);
-      }
-    }
-    testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactionSelectWithPartFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithPartFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = 4000;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      if(file.getLen() < 4000) {
-        String fileName = file.getPath().getName();
-        String startKey = fileName.substring(0, 32);
-        expectedStartKeys.add(startKey);
-      }
-    }
-    // set the mob file compaction mergeable threshold
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-    testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactionSelectWithForceAllFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithForceAllFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = 4000;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      String fileName = file.getPath().getName();
-      String startKey = fileName.substring(0, 32);
-      expectedStartKeys.add(startKey);
-    }
-    // set the mob file compaction mergeable threshold
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-    testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithDefaultBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-    testCompactDelFiles(tableName, 1, 13, false);
-  }
-
-  @Test
-  public void testCompactDelFilesWithSmallBatchSize() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithSmallBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-
-    // set the mob file compaction batch size
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, 4);
-    testCompactDelFiles(tableName, 1, 13, false);
-  }
-
-  @Test
-  public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithSmallBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-
-    // set the max del file count
-    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
-    // set the mob file compaction batch size
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, 2);
-    testCompactDelFiles(tableName, 4, 13, false);
-  }
-
-  /**
-   * Tests the selectFiles
-   * @param tableName the table name
-   * @param type the expected compaction type
-   * @param expected the expected start keys
-   */
-  private void testSelectFiles(String tableName, final CompactionType type,
-    final boolean isForceAllFiles, final List<String> expected) throws IOException {
-    PartitionedMobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs,
-      TableName.valueOf(tableName), hcd, pool) {
-      @Override
-      public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
-        throws IOException {
-        if (files == null || files.isEmpty()) {
-          return null;
-        }
-        PartitionedMobFileCompactionRequest request = select(files, isForceAllFiles);
-        // assert the compaction type
-        Assert.assertEquals(type, request.type);
-        // assert get the right partitions
-        compareCompactedPartitions(expected, request.compactionPartitions);
-        // assert get the right del files
-        compareDelFiles(request.delFiles);
-        return null;
-      }
-    };
-    compactor.compact(allFiles, isForceAllFiles);
-  }
-
-  /**
-   * Tests the compacteDelFile
-   * @param tableName the table name
-   * @param expectedFileCount the expected file count
-   * @param expectedCellCount the expected cell count
-   */
-  private void testCompactDelFiles(String tableName, final int expectedFileCount,
-      final int expectedCellCount, boolean isForceAllFiles) throws IOException {
-    PartitionedMobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs,
-      TableName.valueOf(tableName), hcd, pool) {
-      @Override
-      protected List<Path> performCompaction(PartitionedMobFileCompactionRequest request)
-          throws IOException {
-        List<Path> delFilePaths = new ArrayList<Path>();
-        for (FileStatus delFile : request.delFiles) {
-          delFilePaths.add(delFile.getPath());
-        }
-        List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
-        // assert the del files are merged.
-        Assert.assertEquals(expectedFileCount, newDelPaths.size());
-        Assert.assertEquals(expectedCellCount, countDelCellsInDelFiles(newDelPaths));
-        return null;
-      }
-    };
-    compactor.compact(allFiles, isForceAllFiles);
-  }
-
-  /**
-   * Lists the files in the path
-   */
-  private void listFiles() throws IOException {
-    for (FileStatus file : fs.listStatus(basePath)) {
-      allFiles.add(file);
-      if (file.getPath().getName().endsWith("_del")) {
-        delFiles.add(file);
-      } else {
-        mobFiles.add(file);
-      }
-    }
-  }
-
-  /**
-   * Compares the compacted partitions.
-   * @param partitions the collection of CompactedPartitions
-   */
-  private void compareCompactedPartitions(List<String> expected,
-      Collection<CompactionPartition> partitions) {
-    List<String> actualKeys = new ArrayList<>();
-    for (CompactionPartition partition : partitions) {
-      actualKeys.add(partition.getPartitionId().getStartKey());
-    }
-    Collections.sort(expected);
-    Collections.sort(actualKeys);
-    Assert.assertEquals(expected.size(), actualKeys.size());
-    for (int i = 0; i < expected.size(); i++) {
-      Assert.assertEquals(expected.get(i), actualKeys.get(i));
-    }
-  }
-
-  /**
-   * Compares the del files.
-   * @param allDelFiles all the del files
-   */
-  private void compareDelFiles(Collection<FileStatus> allDelFiles) {
-    int i = 0;
-    for (FileStatus file : allDelFiles) {
-      Assert.assertEquals(delFiles.get(i), file);
-      i++;
-    }
-  }
-
-  /**
-   * Creates store files.
-   * @param basePath the path to create file
-   * @family the family name
-   * @qualifier the column qualifier
-   * @count the store file number
-   * @type the key type
-   */
-  private void createStoreFiles(Path basePath, String family, String qualifier, int count,
-      Type type) throws IOException {
-    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
-    String startKey = "row_";
-    MobFileName mobFileName = null;
-    for (int i = 0; i < count; i++) {
-      byte[] startRow = Bytes.toBytes(startKey + i) ;
-      if(type.equals(Type.Delete)) {
-        mobFileName = MobFileName.create(startRow, MobUtils.formatDate(
-            new Date()), delSuffix);
-      }
-      if(type.equals(Type.Put)){
-        mobFileName = MobFileName.create(Bytes.toBytes(startKey + i), MobUtils.formatDate(
-            new Date()), mobSuffix);
-      }
-      StoreFile.Writer mobFileWriter = new StoreFile.WriterBuilder(conf, cacheConf, fs)
-      .withFileContext(meta).withFilePath(new Path(basePath, mobFileName.getFileName())).build();
-      writeStoreFile(mobFileWriter, startRow, Bytes.toBytes(family), Bytes.toBytes(qualifier),
-          type, (i+1)*1000);
-    }
-  }
-
-  /**
-   * Writes data to store file.
-   * @param writer the store file writer
-   * @param row the row key
-   * @param family the family name
-   * @param qualifier the column qualifier
-   * @param type the key type
-   * @param size the size of value
-   */
-  private static void writeStoreFile(final StoreFile.Writer writer, byte[]row, byte[] family,
-      byte[] qualifier, Type type, int size) throws IOException {
-    long now = System.currentTimeMillis();
-    try {
-      byte[] dummyData = new byte[size];
-      new Random().nextBytes(dummyData);
-      writer.append(new KeyValue(row, family, qualifier, now, type, dummyData));
-    } finally {
-      writer.close();
-    }
-  }
-
-  /**
-   * Gets the number of del cell in the del files
-   * @param paths the del file paths
-   * @return the cell size
-   */
-  private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
-    List<StoreFile> sfs = new ArrayList<StoreFile>();
-    int size = 0;
-    for(Path path : paths) {
-      StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.NONE);
-      sfs.add(sf);
-    }
-    List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
-        false, null, HConstants.LATEST_TIMESTAMP);
-    Scan scan = new Scan();
-    scan.setMaxVersions(hcd.getMaxVersions());
-    long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
-    long ttl = HStore.determineTTLFromFamily(hcd);
-    ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
-    StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
-        scanners, 0L, HConstants.LATEST_TIMESTAMP);
-    List<Cell> results = new ArrayList<>();
-    boolean hasMore = true;
-
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      size += results.size();
-      results.clear();
-    }
-    scanner.close();
-    return size;
-  }
-
-  private static ExecutorService createThreadPool() {
-    int maxThreads = 10;
-    long keepAliveTime = 60;
-    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
-    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
-      TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"),
-      new RejectedExecutionHandler() {
-        @Override
-        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
-          try {
-            // waiting for a thread to pick up instead of throwing exceptions.
-            queue.put(r);
-          } catch (InterruptedException e) {
-            throw new RejectedExecutionException(e);
-          }
-        }
-      });
-    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
-    return pool;
-  }
-
-  /**
-   * Resets the configuration.
-   */
-  private void resetConf() {
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD);
-    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 67fcf96..bc5f494 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -267,22 +267,22 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsCount() {
+  public long getCellsCountCompactedToMob() {
     return 20;
   }
 
   @Override
-  public long getMobCompactedFromMobCellsCount() {
+  public long getCellsCountCompactedFromMob() {
     return 10;
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsSize() {
+  public long getCellsSizeCompactedToMob() {
     return 200;
   }
 
   @Override
-  public long getMobCompactedFromMobCellsSize() {
+  public long getCellsSizeCompactedFromMob() {
     return 100;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
deleted file mode 100644
index 10b680a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
-import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
-import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-/**
- * Test mob compaction
- */
-@Category(MediumTests.class)
-public class TestMobCompaction {
-  @Rule
-  public TestName name = new TestName();
-  static final Log LOG = LogFactory.getLog(TestMobCompaction.class.getName());
-  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-  private Configuration conf = null;
-
-  private HRegion region = null;
-  private HTableDescriptor htd = null;
-  private HColumnDescriptor hcd = null;
-  private long mobCellThreshold = 1000;
-
-  private FileSystem fs;
-
-  private static final byte[] COLUMN_FAMILY = fam1;
-  private final byte[] STARTROW = Bytes.toBytes(START_KEY);
-  private int compactionThreshold;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    UTIL.startMiniCluster(1);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    UTIL.shutdownMiniCluster();
-  }
-
-  private void init(Configuration conf, long mobThreshold) throws Exception {
-    this.conf = conf;
-    this.mobCellThreshold = mobThreshold;
-    HBaseTestingUtility UTIL = new HBaseTestingUtility(conf);
-
-    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
-    htd = UTIL.createTableDescriptor(name.getMethodName());
-    hcd = new HColumnDescriptor(COLUMN_FAMILY);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(mobThreshold);
-    hcd.setMaxVersions(1);
-    htd.modifyFamily(hcd);
-
-    region = UTIL.createLocalHRegion(htd, null, null);
-    fs = FileSystem.get(conf);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    region.close();
-    fs.delete(UTIL.getDataTestDir(), true);
-  }
-
-  /**
-   * During compaction, cells smaller than the threshold won't be affected.
-   */
-  @Test
-  public void testSmallerValue() throws Exception {
-    init(UTIL.getConfiguration(), 500);
-    byte[] dummyData = makeDummyData(300); // smaller than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    // one hfile per row
-    for (int i = 0; i < compactionThreshold; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", 0, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", 0, countMobRows());
-
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count", 0, countMobFiles());
-    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", 0, countMobRows());
-  }
-
-  /**
-   * During compaction, the mob threshold size is changed.
-   */
-  @Test
-  public void testLargerValue() throws Exception {
-    init(UTIL.getConfiguration(), 200);
-    byte[] dummyData = makeDummyData(300); // larger than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    for (int i = 0; i < compactionThreshold; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
-    assertEquals("Before compaction: number of mob cells", compactionThreshold,
-        countMobCellsInMetadata());
-    // Change the threshold larger than the data size
-    region.getTableDesc().getFamily(COLUMN_FAMILY).setMobThreshold(500);
-    region.initialize();
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
-    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", 0, countMobRows());
-  }
-
-  /**
-   * This test will first generate store files, then bulk load them and trigger the compaction. When
-   * compaction, the cell value will be larger than the threshold.
-   */
-  @Test
-  public void testMobCompactionWithBulkload() throws Exception {
-    // The following will produce store files of 600.
-    init(UTIL.getConfiguration(), 300);
-    byte[] dummyData = makeDummyData(600);
-
-    Path hbaseRootDir = FSUtils.getRootDir(conf);
-    Path basedir = new Path(hbaseRootDir, htd.getNameAsString());
-    List<Pair<byte[], String>> hfiles = new ArrayList<>(1);
-    for (int i = 0; i < compactionThreshold; i++) {
-      Path hpath = new Path(basedir, "hfile" + i);
-      hfiles.add(Pair.newPair(COLUMN_FAMILY, hpath.toString()));
-      createHFile(hpath, i, dummyData);
-    }
-
-    // The following will bulk load the above generated store files and compact, with 600(fileSize)
-    // > 300(threshold)
-    boolean result = region.bulkLoadHFiles(hfiles, true, null);
-    assertTrue("Bulkload result:", result);
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", 0, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", 0, countMobRows());
-    assertEquals("Before compaction: referenced mob file count", 0, countReferencedMobFiles());
-
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count:", 1, countMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", compactionThreshold, countMobRows());
-    assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles());
-    assertEquals("After compaction: number of mob cells", compactionThreshold,
-        countMobCellsInMetadata());
-  }
-
-  @Test
-  public void testMajorCompactionAfterDelete() throws Exception {
-    init(UTIL.getConfiguration(), 100);
-    byte[] dummyData = makeDummyData(200); // larger than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    // create hfiles and mob hfiles but don't trigger compaction
-    int numHfiles = compactionThreshold - 1;
-    byte[] deleteRow = Bytes.add(STARTROW, Bytes.toBytes(0));
-    for (int i = 0; i < numHfiles; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", numHfiles, countStoreFiles());
-    assertEquals("Before compaction: mob file count", numHfiles, countMobFiles());
-    assertEquals("Before compaction: rows", numHfiles, countRows());
-    assertEquals("Before compaction: mob rows", numHfiles, countMobRows());
-    assertEquals("Before compaction: number of mob cells", numHfiles, countMobCellsInMetadata());
-    // now let's delete some cells that contain mobs
-    Delete delete = new Delete(deleteRow);
-    delete.addFamily(COLUMN_FAMILY);
-    region.delete(delete);
-    loader.flushcache();
-
-    assertEquals("Before compaction: store files", numHfiles + 1, countStoreFiles());
-    assertEquals("Before compaction: mob files", numHfiles, countMobFiles());
-    // region.compactStores();
-    region.compact(true);
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    // still have original mob hfiles and now added a mob del file
-    assertEquals("After compaction: mob files", numHfiles + 1, countMobFiles());
-
-    Scan scan = new Scan();
-    scan.setRaw(true);
-    InternalScanner scanner = region.getScanner(scan);
-    List<Cell> results = new ArrayList<>();
-    scanner.next(results);
-    int deleteCount = 0;
-    while (!results.isEmpty()) {
-      for (Cell c : results) {
-        if (c.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()) {
-          deleteCount++;
-          assertTrue(Bytes.equals(CellUtil.cloneRow(c), deleteRow));
-        }
-      }
-      results.clear();
-      scanner.next(results);
-    }
-    // assert the delete mark is not retained after the major compaction
-    assertEquals(0, deleteCount);
-    scanner.close();
-    // assert the deleted cell is not counted
-    assertEquals("The cells in mob files", numHfiles - 1, countMobCellsInMobFiles(1));
-  }
-
-  private int countStoreFiles() throws IOException {
-    Store store = region.getStore(COLUMN_FAMILY);
-    return store.getStorefilesCount();
-  }
-
-  private int countMobFiles() throws IOException {
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
-      return files.length;
-    }
-    return 0;
-  }
-
-  private long countMobCellsInMetadata() throws IOException {
-    long mobCellsCount = 0;
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    Configuration copyOfConf = new Configuration(conf);
-    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
-    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        StoreFile sf = new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE);
-        Map<byte[], byte[]> fileInfo = sf.createReader().loadFileInfo();
-        byte[] count = fileInfo.get(StoreFile.MOB_CELLS_COUNT);
-        assertTrue(count != null);
-        mobCellsCount += Bytes.toLong(count);
-      }
-    }
-    return mobCellsCount;
-  }
-
-  private Put createPut(int rowIdx, byte[] dummyData) throws IOException {
-    Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)));
-    p.setDurability(Durability.SKIP_WAL);
-    p.addColumn(COLUMN_FAMILY, Bytes.toBytes("colX"), dummyData);
-    return p;
-  }
-
-  /**
-   * Create an HFile with the given number of bytes
-   */
-  private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException {
-    HFileContext meta = new HFileContextBuilder().build();
-    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
-        .withFileContext(meta).create();
-    long now = System.currentTimeMillis();
-    try {
-      KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY,
-          Bytes.toBytes("colX"), now, dummyData);
-      writer.append(kv);
-    } finally {
-      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
-      writer.close();
-    }
-  }
-
-  private int countMobRows() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    InternalScanner scanner = region.getScanner(scan);
-
-    int scannedCount = 0;
-    List<Cell> results = new ArrayList<>();
-    boolean hasMore = true;
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      for (Cell c : results) {
-        if (MobUtils.isMobReferenceCell(c)) {
-          scannedCount++;
-        }
-      }
-      results.clear();
-    }
-    scanner.close();
-
-    return scannedCount;
-  }
-
-  private int countRows() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    InternalScanner scanner = region.getScanner(scan);
-
-    int scannedCount = 0;
-    List<Cell> results = new ArrayList<Cell>();
-    boolean hasMore = true;
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      scannedCount += results.size();
-      results.clear();
-    }
-    scanner.close();
-
-    return scannedCount;
-  }
-
-  private byte[] makeDummyData(int size) {
-    byte[] dummyData = new byte[size];
-    new Random().nextBytes(dummyData);
-    return dummyData;
-  }
-
-  private int countReferencedMobFiles() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    InternalScanner scanner = region.getScanner(scan);
-
-    List<Cell> kvs = new ArrayList<>();
-    boolean hasMore = true;
-    String fileName;
-    Set<String> files = new HashSet<>();
-    do {
-      kvs.clear();
-      hasMore = scanner.next(kvs);
-      for (Cell kv : kvs) {
-        if (!MobUtils.isMobReferenceCell(kv)) {
-          continue;
-        }
-        if (!MobUtils.hasValidMobRefCellValue(kv)) {
-          continue;
-        }
-        int size = MobUtils.getMobValueLength(kv);
-        if (size <= mobCellThreshold) {
-          continue;
-        }
-        fileName = MobUtils.getMobFileName(kv);
-        if (fileName.isEmpty()) {
-          continue;
-        }
-        files.add(fileName);
-        Path familyPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(),
-            hcd.getNameAsString());
-        assertTrue(fs.exists(new Path(familyPath, fileName)));
-      }
-    } while (hasMore);
-
-    scanner.close();
-
-    return files.size();
-  }
-
-  private int countMobCellsInMobFiles(int expectedNumDelfiles) throws IOException {
-    Configuration copyOfConf = new Configuration(conf);
-    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
-    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    List<StoreFile> sfs = new ArrayList<>();
-    int numDelfiles = 0;
-    int size = 0;
-    if (fs.exists(mobDirPath)) {
-      for (FileStatus f : fs.listStatus(mobDirPath)) {
-        StoreFile sf = new StoreFile(fs, f.getPath(), conf, cacheConfig, BloomType.NONE);
-        sfs.add(sf);
-        if (StoreFileInfo.isDelFile(sf.getPath())) {
-          numDelfiles++;
-        }
-      }
-      List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, null,
-          HConstants.LATEST_TIMESTAMP);
-      Scan scan = new Scan();
-      scan.setMaxVersions(hcd.getMaxVersions());
-      long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
-      long ttl = HStore.determineTTLFromFamily(hcd);
-      ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
-      StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
-          scanners, 0L, HConstants.LATEST_TIMESTAMP);
-      List<Cell> results = new ArrayList<>();
-      boolean hasMore = true;
-      while (hasMore) {
-        hasMore = scanner.next(results);
-        size += results.size();
-        results.clear();
-      }
-    }
-    // assert the number of the existing del files
-    assertEquals(expectedNumDelfiles, numDelfiles);
-    return size;
-  }
-}

[6/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b31a6acf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b31a6acf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b31a6acf

Branch: refs/heads/hbase-11339
Commit: b31a6acf4c24d55ac74eaf4d669a33078e27ef89
Parents: 6388b3b
Author: anoopsjohn <an...@gmail.com>
Authored: Thu May 28 13:43:12 2015 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Thu May 28 13:43:12 2015 +0530

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/client/Admin.java   |   4 +-
 .../apache/hadoop/hbase/client/HBaseAdmin.java  |   4 +-
 .../src/main/resources/hbase-default.xml        |  30 +-
 .../regionserver/MetricsRegionServerSource.java |  16 +-
 .../MetricsRegionServerWrapper.java             |  16 +-
 .../MetricsRegionServerSourceImpl.java          |  16 +-
 .../hbase/chaos/actions/CompactMobAction.java   |   4 +-
 .../org/apache/hadoop/hbase/io/HFileLink.java   |   4 +-
 .../master/ExpiredMobFileCleanerChore.java      |   2 +-
 .../org/apache/hadoop/hbase/master/HMaster.java |  61 +-
 .../hbase/master/MasterMobCompactionThread.java | 184 ++++
 .../master/MasterMobFileCompactionThread.java   | 184 ----
 .../hadoop/hbase/master/MasterRpcServices.java  |  18 +-
 .../hadoop/hbase/master/MobCompactionChore.java |  97 ++
 .../hbase/master/MobFileCompactionChore.java    |  97 --
 .../master/handler/DeleteTableHandler.java      |  14 +-
 .../master/procedure/DeleteTableProcedure.java  |   9 +-
 .../hadoop/hbase/mob/DefaultMobCompactor.java   | 304 ------
 .../hbase/mob/DefaultMobStoreCompactor.java     | 305 ++++++
 .../hbase/mob/DefaultMobStoreFlusher.java       |   4 +-
 .../hadoop/hbase/mob/ExpiredMobFileCleaner.java |   2 +-
 .../apache/hadoop/hbase/mob/MobConstants.java   |  40 +-
 .../apache/hadoop/hbase/mob/MobFileCache.java   |   4 +-
 .../apache/hadoop/hbase/mob/MobFileName.java    |   4 +-
 .../apache/hadoop/hbase/mob/MobStoreEngine.java |   2 +-
 .../org/apache/hadoop/hbase/mob/MobUtils.java   |  62 +-
 .../mob/compactions/MobCompactionRequest.java   |  64 ++
 .../hbase/mob/compactions/MobCompactor.java     |  90 ++
 .../PartitionedMobCompactionRequest.java        | 146 +++
 .../compactions/PartitionedMobCompactor.java    | 655 +++++++++++++
 .../MobFileCompactionRequest.java               |  64 --
 .../mob/filecompactions/MobFileCompactor.java   |  90 --
 .../PartitionedMobFileCompactionRequest.java    | 146 ---
 .../PartitionedMobFileCompactor.java            | 636 -------------
 .../hbase/mob/mapreduce/MemStoreWrapper.java    |  17 +-
 .../hadoop/hbase/mob/mapreduce/SweepJob.java    |  22 +-
 .../hbase/mob/mapreduce/SweepReducer.java       |  81 +-
 .../hadoop/hbase/mob/mapreduce/Sweeper.java     |   4 +-
 .../hadoop/hbase/regionserver/HMobStore.java    |  40 +-
 .../MetricsRegionServerWrapperImpl.java         |  48 +-
 .../regionserver/ReversedMobStoreScanner.java   |   2 +-
 .../hbase/regionserver/StoreFileInfo.java       |   2 +-
 .../hadoop/hbase/snapshot/SnapshotManifest.java |   8 +-
 .../hbase/mob/compactions/TestMobCompactor.java | 924 +++++++++++++++++++
 .../TestPartitionedMobCompactionRequest.java    |  60 ++
 .../TestPartitionedMobCompactor.java            | 440 +++++++++
 .../filecompactions/TestMobFileCompactor.java   | 922 ------------------
 ...TestPartitionedMobFileCompactionRequest.java |  60 --
 .../TestPartitionedMobFileCompactor.java        | 436 ---------
 .../MetricsRegionServerWrapperStub.java         |   8 +-
 .../hbase/regionserver/TestMobCompaction.java   | 460 ---------
 .../regionserver/TestMobStoreCompaction.java    | 460 +++++++++
 hbase-shell/src/main/ruby/hbase/admin.rb        |   4 +-
 53 files changed, 3692 insertions(+), 3684 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 3c0383f..1e50525 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -1461,7 +1461,7 @@ public interface Admin extends Abortable, Closeable {
    * @throws IOException
    * @throws InterruptedException
    */
-  void compactMob(final TableName tableName) throws IOException,
+  void compactMobs(final TableName tableName) throws IOException,
     InterruptedException;
 
   /**
@@ -1482,7 +1482,7 @@ public interface Admin extends Abortable, Closeable {
    * @throws IOException
    * @throws InterruptedException
    */
-  void majorCompactMob(final TableName tableName) throws IOException,
+  void majorCompactMobs(final TableName tableName) throws IOException,
     InterruptedException;
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index caa12d2..4461e5c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -4052,7 +4052,7 @@ public class HBaseAdmin implements Admin {
    * {@inheritDoc}
    */
   @Override
-  public void compactMob(final TableName tableName) throws IOException, InterruptedException {
+  public void compactMobs(final TableName tableName) throws IOException, InterruptedException {
     checkTableNameNotNull(tableName);
     compactMob(tableName, null, false);
   }
@@ -4073,7 +4073,7 @@ public class HBaseAdmin implements Admin {
    * {@inheritDoc}
    */
   @Override
-  public void majorCompactMob(final TableName tableName) throws IOException, InterruptedException {
+  public void majorCompactMobs(final TableName tableName) throws IOException, InterruptedException {
     checkTableNameNotNull(tableName);
     compactMob(tableName, null, true);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 540dded..66f5e73 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -1605,54 +1605,54 @@ possible configurations would overwhelm and obscure the important.
     </description>
   </property>
   <property>
-    <name>hbase.mob.file.compaction.mergeable.threshold</name>
+    <name>hbase.mob.compaction.mergeable.threshold</name>
     <value>201326592</value>
     <description>
       If the size of a mob file is less than this value, it's regarded as a small
-      file and needs to be merged in mob file compaction. The default value is 192MB.
+      file and needs to be merged in mob compaction. The default value is 192MB.
     </description>
   </property>
   <property>
     <name>hbase.mob.delfile.max.count</name>
     <value>3</value>
     <description>
-      The max number of del files that is allowed in the mob file compaction.
-      In the mob file compaction, when the number of existing del files is larger than
+      The max number of del files that is allowed in the mob compaction.
+      In the mob compaction, when the number of existing del files is larger than
       this value, they are merged until number of del files is not larger this value.
       The default value is 3.
     </description>
   </property>
   <property>
-    <name>hbase.mob.file.compaction.batch.size</name>
+    <name>hbase.mob.compaction.batch.size</name>
     <value>100</value>
     <description>
-      The max number of the mob files that is allowed in a batch of the mob file compaction.
-      The mob file compaction merges the small mob files to bigger ones. If the number of the
+      The max number of the mob files that is allowed in a batch of the mob compaction.
+      The mob compaction merges the small mob files to bigger ones. If the number of the
       small files is very large, it could lead to a "too many opened file handlers" in the merge.
       And the merge has to be split into batches. This value limits the number of mob files
-      that are selected in a batch of the mob file compaction. The default value is 100.
+      that are selected in a batch of the mob compaction. The default value is 100.
     </description>
   </property>
   <property>
-    <name>hbase.mob.file.compaction.chore.period</name>
+    <name>hbase.mob.compaction.chore.period</name>
     <value>604800</value>
     <description>
-      The period that MobFileCompactionChore runs. The unit is second.
+      The period that MobCompactionChore runs. The unit is second.
       The default value is one week.
     </description>
   </property>
   <property>
-    <name>hbase.mob.file.compactor.class</name>
-    <value>org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactor</value>
+    <name>hbase.mob.compactor.class</name>
+    <value>org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor</value>
     <description>
-      Implementation of mob file compactor, the default one is PartitionedMobFileCompactor.
+      Implementation of mob compactor, the default one is PartitionedMobCompactor.
     </description>
   </property>
   <property>
-    <name>hbase.mob.file.compaction.threads.max</name>
+    <name>hbase.mob.compaction.threads.max</name>
     <value>1</value>
     <description>
-      The max number of threads used in MobFileCompactor.
+      The max number of threads used in MobCompactor.
     </description>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 268d4af..80b5cd2 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -267,17 +267,17 @@ public interface MetricsRegionServerSource extends BaseSource {
   String MAJOR_COMPACTED_CELLS_SIZE = "majorCompactedCellsSize";
   String MAJOR_COMPACTED_CELLS_SIZE_DESC =
       "The total amount of data processed during major compactions, in bytes";
-  String MOB_COMPACTED_INTO_MOB_CELLS_COUNT = "mobCompactedIntoMobCellsCount";
-  String MOB_COMPACTED_INTO_MOB_CELLS_COUNT_DESC =
+  String CELLS_COUNT_COMPACTED_TO_MOB = "cellsCountCompactedToMob";
+  String CELLS_COUNT_COMPACTED_TO_MOB_DESC =
       "The number of cells moved to mob during compaction";
-  String MOB_COMPACTED_FROM_MOB_CELLS_COUNT = "mobCompactedFromMobCellsCount";
-  String MOB_COMPACTED_FROM_MOB_CELLS_COUNT_DESC =
+  String CELLS_COUNT_COMPACTED_FROM_MOB = "cellsCountCompactedFromMob";
+  String CELLS_COUNT_COMPACTED_FROM_MOB_DESC =
       "The number of cells moved from mob during compaction";
-  String MOB_COMPACTED_INTO_MOB_CELLS_SIZE = "mobCompactedIntoMobCellsSize";
-  String MOB_COMPACTED_INTO_MOB_CELLS_SIZE_DESC =
+  String CELLS_SIZE_COMPACTED_TO_MOB = "cellsSizeCompactedToMob";
+  String CELLS_SIZE_COMPACTED_TO_MOB_DESC =
       "The total amount of cells move to mob during compaction, in bytes";
-  String MOB_COMPACTED_FROM_MOB_CELLS_SIZE = "mobCompactedFromMobCellsSize";
-  String MOB_COMPACTED_FROM_MOB_CELLS_SIZE_DESC =
+  String CELLS_SIZE_COMPACTED_FROM_MOB = "cellsSizeCompactedFromMob";
+  String CELLS_SIZE_COMPACTED_FROM_MOB_DESC =
       "The total amount of cells move from mob during compaction, in bytes";
   String MOB_FLUSH_COUNT = "mobFlushCount";
   String MOB_FLUSH_COUNT_DESC = "The number of the flushes in mob-enabled stores";

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index b609b4a..f2bd8ff 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -258,24 +258,24 @@ public interface MetricsRegionServerWrapper {
   long getMajorCompactedCellsSize();
 
   /**
-   * Gets the number of cells move to mob during compaction.
+   * Gets the number of cells moved to mob during compaction.
    */
-  long getMobCompactedIntoMobCellsCount();
+  long getCellsCountCompactedToMob();
 
   /**
-   * Gets the number of cells move from mob during compaction.
+   * Gets the number of cells moved from mob during compaction.
    */
-  long getMobCompactedFromMobCellsCount();
+  long getCellsCountCompactedFromMob();
 
   /**
-   * Gets the total amount of cells move to mob during compaction, in bytes.
+   * Gets the total amount of cells moved to mob during compaction, in bytes.
    */
-  long getMobCompactedIntoMobCellsSize();
+  long getCellsSizeCompactedToMob();
 
   /**
-   * Gets the total amount of cells move from mob during compaction, in bytes.
+   * Gets the total amount of cells moved from mob during compaction, in bytes.
    */
-  long getMobCompactedFromMobCellsSize();
+  long getCellsSizeCompactedFromMob();
 
   /**
    * Gets the number of the flushes in mob-enabled stores.

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index cadb574..26c55bb 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -259,14 +259,14 @@ public class MetricsRegionServerSourceImpl
           .addCounter(Interns.info(MAJOR_COMPACTED_CELLS_SIZE, MAJOR_COMPACTED_CELLS_SIZE_DESC),
               rsWrap.getMajorCompactedCellsSize())
 
-          .addCounter(Interns.info(MOB_COMPACTED_FROM_MOB_CELLS_COUNT, MOB_COMPACTED_FROM_MOB_CELLS_COUNT_DESC),
-              rsWrap.getMobCompactedFromMobCellsCount())
-          .addCounter(Interns.info(MOB_COMPACTED_INTO_MOB_CELLS_COUNT, MOB_COMPACTED_INTO_MOB_CELLS_COUNT_DESC),
-              rsWrap.getMobCompactedIntoMobCellsCount())
-          .addCounter(Interns.info(MOB_COMPACTED_FROM_MOB_CELLS_SIZE, MOB_COMPACTED_FROM_MOB_CELLS_SIZE_DESC),
-              rsWrap.getMobCompactedFromMobCellsSize())
-          .addCounter(Interns.info(MOB_COMPACTED_INTO_MOB_CELLS_SIZE, MOB_COMPACTED_INTO_MOB_CELLS_SIZE_DESC),
-              rsWrap.getMobCompactedIntoMobCellsSize())
+          .addCounter(Interns.info(CELLS_COUNT_COMPACTED_FROM_MOB, CELLS_COUNT_COMPACTED_FROM_MOB_DESC),
+              rsWrap.getCellsCountCompactedFromMob())
+          .addCounter(Interns.info(CELLS_COUNT_COMPACTED_TO_MOB, CELLS_COUNT_COMPACTED_TO_MOB_DESC),
+              rsWrap.getCellsCountCompactedToMob())
+          .addCounter(Interns.info(CELLS_SIZE_COMPACTED_FROM_MOB, CELLS_SIZE_COMPACTED_FROM_MOB_DESC),
+              rsWrap.getCellsSizeCompactedFromMob())
+          .addCounter(Interns.info(CELLS_SIZE_COMPACTED_TO_MOB, CELLS_SIZE_COMPACTED_TO_MOB_DESC),
+              rsWrap.getCellsSizeCompactedToMob())
           .addCounter(Interns.info(MOB_FLUSH_COUNT, MOB_FLUSH_COUNT_DESC),
               rsWrap.getMobFlushCount())
           .addCounter(Interns.info(MOB_FLUSHED_CELLS_COUNT, MOB_FLUSHED_CELLS_COUNT_DESC),

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
index a349d3d..87c6dee 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactMobAction.java
@@ -51,9 +51,9 @@ public class CompactMobAction extends Action {
     LOG.info("Performing action: Compact mob of table " + tableName + ", major=" + major);
     try {
       if (major) {
-        admin.majorCompactMob(tableName);
+        admin.majorCompactMobs(tableName);
       } else {
-        admin.compactMob(tableName);
+        admin.compactMobs(tableName);
       }
     } catch (Exception ex) {
       LOG.warn("Mob Compaction failed, might be caused by other chaos: " + ex.getMessage());

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
index a950dce..d070539 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -184,7 +184,9 @@ public class HFileLink extends FileLink {
   /**
    * @return the path of the mob hfiles.
    */
-  public Path getMobPath() { return this.mobPath; }
+  public Path getMobPath() {
+    return this.mobPath;
+  }
 
     /**
    * @param path Path to check.

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java
index 7b06462..7ca3362 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ExpiredMobFileCleanerChore.java
@@ -92,7 +92,7 @@ public class ExpiredMobFileCleanerChore extends ScheduledChore {
                   lock.release();
                 } catch (IOException e) {
                   LOG.error(
-                    "Fail to release the write lock for the table " + htd.getNameAsString(), e);
+                    "Fail to release the read lock for the table " + htd.getNameAsString(), e);
                 }
               }
             }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index b3fc2a1..fc9aac7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -110,7 +110,6 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
 import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager;
 import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
 import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
@@ -280,13 +279,13 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
   private LogCleaner logCleaner;
   private HFileCleaner hfileCleaner;
   private ExpiredMobFileCleanerChore expiredMobFileCleanerChore;
-  private MobFileCompactionChore mobFileCompactChore;
-  MasterMobFileCompactionThread mobFileCompactThread;
-  // used to synchronize the mobFileCompactionStates
-  private final IdLock mobFileCompactionLock = new IdLock();
-  // save the information of mob file compactions in tables.
+  private MobCompactionChore mobCompactChore;
+  private MasterMobCompactionThread mobCompactThread;
+  // used to synchronize the mobCompactionStates
+  private final IdLock mobCompactionLock = new IdLock();
+  // save the information of mob compactions in tables.
   // the key is table name, the value is the number of compactions in that table.
-  private Map<TableName, AtomicInteger> mobFileCompactionStates = Maps.newConcurrentMap();
+  private Map<TableName, AtomicInteger> mobCompactionStates = Maps.newConcurrentMap();
 
   MasterCoprocessorHost cpHost;
 
@@ -796,9 +795,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     this.expiredMobFileCleanerChore = new ExpiredMobFileCleanerChore(this);
     getChoreService().scheduleChore(expiredMobFileCleanerChore);
 
-    this.mobFileCompactChore = new MobFileCompactionChore(this);
-    getChoreService().scheduleChore(mobFileCompactChore);
-    this.mobFileCompactThread = new MasterMobFileCompactionThread(this);
+    this.mobCompactChore = new MobCompactionChore(this);
+    getChoreService().scheduleChore(mobCompactChore);
+    this.mobCompactThread = new MasterMobCompactionThread(this);
 
     if (this.cpHost != null) {
       // don't let cp initialization errors kill the master
@@ -1134,8 +1133,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     if (this.expiredMobFileCleanerChore != null) {
       this.expiredMobFileCleanerChore.cancel(true);
     }
-    if (this.mobFileCompactChore != null) {
-      this.mobFileCompactChore.cancel(true);
+    if (this.mobCompactChore != null) {
+      this.mobCompactChore.cancel(true);
     }
     if (this.balancerChore != null) {
       this.balancerChore.cancel(true);
@@ -1149,8 +1148,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
     if (this.clusterStatusPublisherChore != null){
       clusterStatusPublisherChore.cancel(true);
     }
-    if (this.mobFileCompactThread != null) {
-      this.mobFileCompactThread.close();
+    if (this.mobCompactThread != null) {
+      this.mobCompactThread.close();
     }
   }
 
@@ -2453,50 +2452,62 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
    * @return If a given table is in mob file compaction now.
    */
   public CompactionState getMobCompactionState(TableName tableName) {
-    AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+    AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
     if (compactionsCount != null && compactionsCount.get() != 0) {
       return CompactionState.MAJOR_AND_MINOR;
     }
     return CompactionState.NONE;
   }
 
-  public void reportMobFileCompactionStart(TableName tableName) throws IOException {
+  public void reportMobCompactionStart(TableName tableName) throws IOException {
     IdLock.Entry lockEntry = null;
     try {
-      lockEntry = mobFileCompactionLock.getLockEntry(tableName.hashCode());
-      AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
+      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
       if (compactionsCount == null) {
         compactionsCount = new AtomicInteger(0);
-        mobFileCompactionStates.put(tableName, compactionsCount);
+        mobCompactionStates.put(tableName, compactionsCount);
       }
       compactionsCount.incrementAndGet();
     } finally {
       if (lockEntry != null) {
-        mobFileCompactionLock.releaseLockEntry(lockEntry);
+        mobCompactionLock.releaseLockEntry(lockEntry);
       }
     }
   }
 
-  public void reportMobFileCompactionEnd(TableName tableName) throws IOException {
+  public void reportMobCompactionEnd(TableName tableName) throws IOException {
     IdLock.Entry lockEntry = null;
     try {
-      lockEntry = mobFileCompactionLock.getLockEntry(tableName.hashCode());
-      AtomicInteger compactionsCount = mobFileCompactionStates.get(tableName);
+      lockEntry = mobCompactionLock.getLockEntry(tableName.hashCode());
+      AtomicInteger compactionsCount = mobCompactionStates.get(tableName);
       if (compactionsCount != null) {
         int count = compactionsCount.decrementAndGet();
         // remove the entry if the count is 0.
         if (count == 0) {
-          mobFileCompactionStates.remove(tableName);
+          mobCompactionStates.remove(tableName);
         }
       }
     } finally {
       if (lockEntry != null) {
-        mobFileCompactionLock.releaseLockEntry(lockEntry);
+        mobCompactionLock.releaseLockEntry(lockEntry);
       }
     }
   }
 
   /**
+   * Requests mob compaction.
+   * @param tableName The table the compact.
+   * @param columns The compacted columns.
+   * @param allFiles Whether add all mob files into the compaction.
+   */
+  public void requestMobCompaction(TableName tableName,
+    List<HColumnDescriptor> columns, boolean allFiles) throws IOException {
+    mobCompactThread.requestMobCompaction(conf, fs, tableName, columns,
+      tableLockManager, allFiles);
+  }
+
+  /**
    * Queries the state of the {@link LoadBalancerTracker}. If the balancer is not initialized,
    * false is returned.
    *

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobCompactionThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobCompactionThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobCompactionThread.java
new file mode 100644
index 0000000..d1f58ba
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobCompactionThread.java
@@ -0,0 +1,184 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+/**
+ * The mob compaction thread used in {@link MasterRpcServices}
+ */
+@InterfaceAudience.Private
+public class MasterMobCompactionThread {
+  static final Log LOG = LogFactory.getLog(MasterMobCompactionThread.class);
+  private final HMaster master;
+  private final Configuration conf;
+  private final ExecutorService mobCompactorPool;
+  private final ExecutorService masterMobPool;
+
+  public MasterMobCompactionThread(HMaster master) {
+    this.master = master;
+    this.conf = master.getConfiguration();
+    final String n = Thread.currentThread().getName();
+    // this pool is used to run the mob compaction
+    this.masterMobPool = new ThreadPoolExecutor(1, 2, 60, TimeUnit.SECONDS,
+      new SynchronousQueue<Runnable>(), new ThreadFactory() {
+        @Override
+        public Thread newThread(Runnable r) {
+          Thread t = new Thread(r);
+          t.setName(n + "-MasterMobCompaction-" + EnvironmentEdgeManager.currentTime());
+          return t;
+        }
+      });
+    ((ThreadPoolExecutor) this.masterMobPool).allowCoreThreadTimeOut(true);
+    // this pool is used in the mob compaction to compact the mob files by partitions
+    // in parallel
+    this.mobCompactorPool = MobUtils
+      .createMobCompactorThreadPool(master.getConfiguration());
+  }
+
+  /**
+   * Requests mob compaction
+   * @param conf The Configuration
+   * @param fs The file system
+   * @param tableName The table the compact
+   * @param columns The column descriptors
+   * @param tableLockManager The tableLock manager
+   * @param allFiles Whether add all mob files into the compaction.
+   */
+  public void requestMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
+    List<HColumnDescriptor> columns, TableLockManager tableLockManager, boolean allFiles)
+    throws IOException {
+    master.reportMobCompactionStart(tableName);
+    try {
+      masterMobPool.execute(new CompactionRunner(fs, tableName, columns, tableLockManager,
+        allFiles, mobCompactorPool));
+    } catch (RejectedExecutionException e) {
+      // in case the request is rejected by the pool
+      try {
+        master.reportMobCompactionEnd(tableName);
+      } catch (IOException e1) {
+        LOG.error("Failed to mark end of mob compation", e1);
+      }
+      throw e;
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("The mob compaction is requested for the columns " + columns
+        + " of the table " + tableName.getNameAsString());
+    }
+  }
+
+  private class CompactionRunner implements Runnable {
+    private FileSystem fs;
+    private TableName tableName;
+    private List<HColumnDescriptor> hcds;
+    private TableLockManager tableLockManager;
+    private boolean allFiles;
+    private ExecutorService pool;
+
+    public CompactionRunner(FileSystem fs, TableName tableName, List<HColumnDescriptor> hcds,
+      TableLockManager tableLockManager, boolean allFiles, ExecutorService pool) {
+      super();
+      this.fs = fs;
+      this.tableName = tableName;
+      this.hcds = hcds;
+      this.tableLockManager = tableLockManager;
+      this.allFiles = allFiles;
+      this.pool = pool;
+    }
+
+    @Override
+    public void run() {
+      try {
+        for (HColumnDescriptor hcd : hcds) {
+          MobUtils.doMobCompaction(conf, fs, tableName, hcd, pool, tableLockManager,
+            allFiles);
+        }
+      } catch (IOException e) {
+        LOG.error("Failed to perform the mob compaction", e);
+      } finally {
+        try {
+          master.reportMobCompactionEnd(tableName);
+        } catch (IOException e) {
+          LOG.error("Failed to mark end of mob compation", e);
+        }
+      }
+    }
+  }
+
+  /**
+   * Only interrupt once it's done with a run through the work loop.
+   */
+  private void interruptIfNecessary() {
+    mobCompactorPool.shutdown();
+    masterMobPool.shutdown();
+  }
+
+  /**
+   * Wait for all the threads finish.
+   */
+  private void join() {
+    waitFor(mobCompactorPool, "Mob Compaction Thread");
+    waitFor(masterMobPool, "Region Server Mob Compaction Thread");
+  }
+
+  /**
+   * Closes the MasterMobCompactionThread.
+   */
+  public void close() {
+    interruptIfNecessary();
+    join();
+  }
+
+  /**
+   * Wait for thread finish.
+   * @param t the thread to wait
+   * @param name the thread name.
+   */
+  private void waitFor(ExecutorService t, String name) {
+    boolean done = false;
+    while (!done) {
+      try {
+        done = t.awaitTermination(60, TimeUnit.SECONDS);
+        LOG.info("Waiting for " + name + " to finish...");
+        if (!done) {
+          t.shutdownNow();
+        }
+      } catch (InterruptedException ie) {
+        LOG.warn("Interrupted waiting for " + name + " to finish...");
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobFileCompactionThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobFileCompactionThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobFileCompactionThread.java
deleted file mode 100644
index f6810a1..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMobFileCompactionThread.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-/**
- * The mob file compaction thread used in {@link MasterRpcServices}
- */
-@InterfaceAudience.Private
-public class MasterMobFileCompactionThread {
-  static final Log LOG = LogFactory.getLog(MasterMobFileCompactionThread.class);
-  private final HMaster master;
-  private final Configuration conf;
-  private final ExecutorService mobFileCompactorPool;
-  private final ExecutorService masterMobPool;
-
-  public MasterMobFileCompactionThread(HMaster master) {
-    this.master = master;
-    this.conf = master.getConfiguration();
-    final String n = Thread.currentThread().getName();
-    // this pool is used to run the mob file compaction
-    this.masterMobPool = new ThreadPoolExecutor(1, 2, 60, TimeUnit.SECONDS,
-      new SynchronousQueue<Runnable>(), new ThreadFactory() {
-        @Override
-        public Thread newThread(Runnable r) {
-          Thread t = new Thread(r);
-          t.setName(n + "-MasterMobFileCompaction-" + EnvironmentEdgeManager.currentTime());
-          return t;
-        }
-      });
-    ((ThreadPoolExecutor) this.masterMobPool).allowCoreThreadTimeOut(true);
-    // this pool is used in the mob file compaction to compact the mob files by partitions
-    // in parallel
-    this.mobFileCompactorPool = MobUtils
-      .createMobFileCompactorThreadPool(master.getConfiguration());
-  }
-
-  /**
-   * Requests mob file compaction
-   * @param conf The Configuration
-   * @param fs The file system
-   * @param tableName The table the compact
-   * @param hcds The column descriptors
-   * @param tableLockManager The tableLock manager
-   * @param isForceAllFiles Whether add all mob files into the compaction.
-   */
-  public void requestMobFileCompaction(Configuration conf, FileSystem fs, TableName tableName,
-    List<HColumnDescriptor> hcds, TableLockManager tableLockManager, boolean isForceAllFiles)
-    throws IOException {
-    master.reportMobFileCompactionStart(tableName);
-    try {
-      masterMobPool.execute(new CompactionRunner(fs, tableName, hcds, tableLockManager,
-        isForceAllFiles, mobFileCompactorPool));
-    } catch (RejectedExecutionException e) {
-      // in case the request is rejected by the pool
-      try {
-        master.reportMobFileCompactionEnd(tableName);
-      } catch (IOException e1) {
-        LOG.error("Failed to mark end of mob file compation", e1);
-      }
-      throw e;
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("The mob file compaction is requested for the columns " + hcds + " of the table "
-        + tableName.getNameAsString());
-    }
-  }
-
-  private class CompactionRunner implements Runnable {
-    private FileSystem fs;
-    private TableName tableName;
-    private List<HColumnDescriptor> hcds;
-    private TableLockManager tableLockManager;
-    private boolean isForceAllFiles;
-    private ExecutorService pool;
-
-    public CompactionRunner(FileSystem fs, TableName tableName, List<HColumnDescriptor> hcds,
-      TableLockManager tableLockManager, boolean isForceAllFiles, ExecutorService pool) {
-      super();
-      this.fs = fs;
-      this.tableName = tableName;
-      this.hcds = hcds;
-      this.tableLockManager = tableLockManager;
-      this.isForceAllFiles = isForceAllFiles;
-      this.pool = pool;
-    }
-
-    @Override
-    public void run() {
-      try {
-        for (HColumnDescriptor hcd : hcds) {
-          MobUtils.doMobFileCompaction(conf, fs, tableName, hcd, pool, tableLockManager,
-            isForceAllFiles);
-        }
-      } catch (IOException e) {
-        LOG.error("Failed to perform the mob file compaction", e);
-      } finally {
-        try {
-          master.reportMobFileCompactionEnd(tableName);
-        } catch (IOException e) {
-          LOG.error("Failed to mark end of mob file compation", e);
-        }
-      }
-    }
-  }
-
-  /**
-   * Only interrupt once it's done with a run through the work loop.
-   */
-  private void interruptIfNecessary() {
-    mobFileCompactorPool.shutdown();
-    masterMobPool.shutdown();
-  }
-
-  /**
-   * Wait for all the threads finish.
-   */
-  private void join() {
-    waitFor(mobFileCompactorPool, "Mob file Compaction Thread");
-    waitFor(masterMobPool, "Region Server Mob File Compaction Thread");
-  }
-
-  /**
-   * Closes the MasterMobFileCompactionThread.
-   */
-  public void close() {
-    interruptIfNecessary();
-    join();
-  }
-
-  /**
-   * Wait for thread finish.
-   * @param t the thread to wait
-   * @param name the thread name.
-   */
-  private void waitFor(ExecutorService t, String name) {
-    boolean done = false;
-    while (!done) {
-      try {
-        done = t.awaitTermination(60, TimeUnit.SECONDS);
-        LOG.info("Waiting for " + name + " to finish...");
-        if (!done) {
-          t.shutdownNow();
-        }
-      } catch (InterruptedException ie) {
-        LOG.warn("Interrupted waiting for " + name + " to finish...");
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 4b8d6b8..7d025db 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -1428,7 +1428,7 @@ public class MasterRpcServices extends RSRpcServices
     if (!master.getTableStateManager().isTableState(tableName, TableState.State.ENABLED)) {
       throw new DoNotRetryIOException("Table " + tableName + " is not enabled");
     }
-    boolean isForceAllFiles = false;
+    boolean allFiles = false;
     List<HColumnDescriptor> compactedColumns = new ArrayList<HColumnDescriptor>();
     HColumnDescriptor[] hcds = master.getTableDescriptors().get(tableName).getColumnFamilies();
     byte[] family = null;
@@ -1437,8 +1437,8 @@ public class MasterRpcServices extends RSRpcServices
       for (HColumnDescriptor hcd : hcds) {
         if (Bytes.equals(family, hcd.getName())) {
           if (!hcd.isMobEnabled()) {
-            LOG.error("Column family " + hcd.getName() + " is not a mob column family");
-            throw new DoNotRetryIOException("Column family " + hcd.getName()
+            LOG.error("Column family " + hcd.getNameAsString() + " is not a mob column family");
+            throw new DoNotRetryIOException("Column family " + hcd.getNameAsString()
                     + " is not a mob column family");
           }
           compactedColumns.add(hcd);
@@ -1452,21 +1452,19 @@ public class MasterRpcServices extends RSRpcServices
       }
     }
     if (compactedColumns.isEmpty()) {
-      LOG.error("No mob column families are assigned in the mob file compaction");
+      LOG.error("No mob column families are assigned in the mob compaction");
       throw new DoNotRetryIOException(
-              "No mob column families are assigned in the mob file compaction");
+              "No mob column families are assigned in the mob compaction");
     }
     if (request.hasMajor() && request.getMajor()) {
-      isForceAllFiles = true;
+      allFiles = true;
     }
     String familyLogMsg = (family != null) ? Bytes.toString(family) : "";
     if (LOG.isTraceEnabled()) {
-      LOG.trace("User-triggered mob file compaction requested for table: "
+      LOG.trace("User-triggered mob compaction requested for table: "
               + tableName.getNameAsString() + " for column family: " + familyLogMsg);
     }
-    master.mobFileCompactThread.requestMobFileCompaction(master.getConfiguration(),
-            master.getFileSystem(), tableName, compactedColumns,
-            master.getTableLockManager(), isForceAllFiles);
+    master.requestMobCompaction(tableName, compactedColumns, allFiles);
     return CompactRegionResponse.newBuilder().build();
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
new file mode 100644
index 0000000..28af3eb
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobCompactionChore.java
@@ -0,0 +1,97 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ScheduledChore;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+/**
+ * The Class MobCompactChore for running compaction regularly to merge small mob files.
+ */
+@InterfaceAudience.Private
+public class MobCompactionChore extends ScheduledChore {
+
+  private static final Log LOG = LogFactory.getLog(MobCompactionChore.class);
+  private HMaster master;
+  private TableLockManager tableLockManager;
+  private ExecutorService pool;
+
+  public MobCompactionChore(HMaster master) {
+    super(master.getServerName() + "-MobCompactionChore", master, master.getConfiguration()
+      .getInt(MobConstants.MOB_COMPACTION_CHORE_PERIOD,
+        MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD), master.getConfiguration().getInt(
+      MobConstants.MOB_COMPACTION_CHORE_PERIOD,
+      MobConstants.DEFAULT_MOB_COMPACTION_CHORE_PERIOD), TimeUnit.SECONDS);
+    this.master = master;
+    this.tableLockManager = master.getTableLockManager();
+    this.pool = MobUtils.createMobCompactorThreadPool(master.getConfiguration());
+  }
+
+  @Override
+  protected void chore() {
+    try {
+      TableDescriptors htds = master.getTableDescriptors();
+      Map<String, HTableDescriptor> map = htds.getAll();
+      for (HTableDescriptor htd : map.values()) {
+        if (!master.getTableStateManager().isTableState(htd.getTableName(),
+          TableState.State.ENABLED)) {
+          continue;
+        }
+        boolean reported = false;
+        try {
+          for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
+            if (!hcd.isMobEnabled()) {
+              continue;
+            }
+            if (!reported) {
+              master.reportMobCompactionStart(htd.getTableName());
+              reported = true;
+            }
+            MobUtils.doMobCompaction(master.getConfiguration(), master.getFileSystem(),
+              htd.getTableName(), hcd, pool, tableLockManager, false);
+          }
+        } finally {
+          if (reported) {
+            master.reportMobCompactionEnd(htd.getTableName());
+          }
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("Failed to compact mob files", e);
+    }
+  }
+
+  @Override
+  protected void cleanup() {
+    super.cleanup();
+    pool.shutdown();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
deleted file mode 100644
index 13c52f0..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MobFileCompactionChore.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.master;
-
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ScheduledChore;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-
-/**
- * The Class MobFileCompactChore for running compaction regularly to merge small mob files.
- */
-@InterfaceAudience.Private
-public class MobFileCompactionChore extends ScheduledChore {
-
-  private static final Log LOG = LogFactory.getLog(MobFileCompactionChore.class);
-  private HMaster master;
-  private TableLockManager tableLockManager;
-  private ExecutorService pool;
-
-  public MobFileCompactionChore(HMaster master) {
-    super(master.getServerName() + "-MobFileCompactChore", master, master.getConfiguration()
-      .getInt(MobConstants.MOB_FILE_COMPACTION_CHORE_PERIOD,
-        MobConstants.DEFAULT_MOB_FILE_COMPACTION_CHORE_PERIOD), master.getConfiguration().getInt(
-      MobConstants.MOB_FILE_COMPACTION_CHORE_PERIOD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_CHORE_PERIOD), TimeUnit.SECONDS);
-    this.master = master;
-    this.tableLockManager = master.getTableLockManager();
-    this.pool = MobUtils.createMobFileCompactorThreadPool(master.getConfiguration());
-  }
-
-  @Override
-  protected void chore() {
-    try {
-      TableDescriptors htds = master.getTableDescriptors();
-      Map<String, HTableDescriptor> map = htds.getAll();
-      for (HTableDescriptor htd : map.values()) {
-        if (!master.getTableStateManager().isTableState(htd.getTableName(),
-          TableState.State.ENABLED)) {
-          continue;
-        }
-        boolean reported = false;
-        try {
-          for (HColumnDescriptor hcd : htd.getColumnFamilies()) {
-            if (!hcd.isMobEnabled()) {
-              continue;
-            }
-            if (!reported) {
-              master.reportMobFileCompactionStart(htd.getTableName());
-              reported = true;
-            }
-            MobUtils.doMobFileCompaction(master.getConfiguration(), master.getFileSystem(),
-              htd.getTableName(), hcd, pool, tableLockManager, false);
-          }
-        } finally {
-          if (reported) {
-            master.reportMobFileCompactionEnd(htd.getTableName());
-          }
-        }
-      }
-    } catch (Exception e) {
-      LOG.error("Fail to clean the expired mob files", e);
-    }
-  }
-
-  @Override
-  protected void cleanup() {
-    super.cleanup();
-    pool.shutdown();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
index cbff5dd..6069eba 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.master.RegionStates;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSUtils;
 
 @InterfaceAudience.Private
@@ -73,8 +74,8 @@ public class DeleteTableHandler extends TableEventHandler {
     long waitTime = server.getConfiguration().
       getLong("hbase.master.wait.on.region", 5 * 60 * 1000);
     for (HRegionInfo region : regions) {
-      long done = System.currentTimeMillis() + waitTime;
-      while (System.currentTimeMillis() < done) {
+      long done = EnvironmentEdgeManager.currentTime() + waitTime;
+      while (EnvironmentEdgeManager.currentTime() < done) {
         if (states.isRegionInState(region, State.FAILED_OPEN)) {
           am.regionOffline(region);
         }
@@ -192,14 +193,7 @@ public class DeleteTableHandler extends TableEventHandler {
       }
 
       // Archive the mob data if there is a mob-enabled column
-      HColumnDescriptor[] hcds = hTableDescriptor.getColumnFamilies();
-      boolean hasMob = false;
-      for (HColumnDescriptor hcd : hcds) {
-        if (hcd.isMobEnabled()) {
-          hasMob = true;
-          break;
-        }
-      }
+      boolean hasMob = MobUtils.hasMobColumns(hTableDescriptor);
       Path mobTableDir = null;
       if (hasMob) {
         // Archive mob data

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index dfc5762..0e561d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -344,14 +344,7 @@ public class DeleteTableProcedure
 
     // Archive the mob data if there is a mob-enabled column
     HTableDescriptor htd = env.getMasterServices().getTableDescriptors().get(tableName);
-    HColumnDescriptor[] hcds = htd.getColumnFamilies();
-    boolean hasMob = false;
-    for (HColumnDescriptor hcd : hcds) {
-      if (hcd.isMobEnabled()) {
-        hasMob = true;
-        break;
-      }
-    }
+    boolean hasMob = MobUtils.hasMobColumns(htd);
     Path mobTableDir = null;
     if (hasMob) {
       // Archive mob data

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
deleted file mode 100644
index d54dca4..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobCompactor.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.hbase.TagType;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Compact passed set of files in the mob-enabled column family.
- */
-@InterfaceAudience.Private
-public class DefaultMobCompactor extends DefaultCompactor {
-
-  private static final Log LOG = LogFactory.getLog(DefaultMobCompactor.class);
-  private long mobSizeThreshold;
-  private HMobStore mobStore;
-  public DefaultMobCompactor(Configuration conf, Store store) {
-    super(conf, store);
-    // The mob cells reside in the mob-enabled column family which is held by HMobStore.
-    // During the compaction, the compactor reads the cells from the mob files and
-    // probably creates new mob files. All of these operations are included in HMobStore,
-    // so we need to cast the Store to HMobStore.
-    if (!(store instanceof HMobStore)) {
-      throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
-    }
-    mobStore = (HMobStore) store;
-    mobSizeThreshold = store.getFamily().getMobThreshold();
-  }
-
-  /**
-   * Creates a writer for a new file in a temporary directory.
-   * @param fd The file details.
-   * @param smallestReadPoint The smallest mvcc readPoint across all the scanners in this region.
-   * @return Writer for a new StoreFile in the tmp dir.
-   * @throws IOException
-   */
-  @Override
-  protected Writer createTmpWriter(FileDetails fd, long smallestReadPoint) throws IOException {
-    // make this writer with tags always because of possible new cells with tags.
-    StoreFile.Writer writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression,
-        true, fd.maxMVCCReadpoint >= smallestReadPoint, true);
-    return writer;
-  }
-
-  @Override
-  protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
-      ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException {
-    Scan scan = new Scan();
-    scan.setMaxVersions(store.getFamily().getMaxVersions());
-    if (scanType == ScanType.COMPACT_DROP_DELETES) {
-      scanType = ScanType.COMPACT_RETAIN_DELETES;
-      return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
-          scanType, smallestReadPoint, earliestPutTs, true);
-    } else {
-      return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
-          scanType, smallestReadPoint, earliestPutTs, false);
-    }
-  }
-
-  // TODO refactor to take advantage of the throughput controller.
-
-  /**
-   * Performs compaction on a column family with the mob flag enabled.
-   * This is for when the mob threshold size has changed or if the mob
-   * column family mode has been toggled via an alter table statement.
-   * Compacts the files by the following rules.
-   * 1. If the cell has a mob reference tag, the cell's value is the path of the mob file.
-   * <ol>
-   * <li>
-   * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
-   * directly copy the (with mob tag) cell into the new store file.
-   * </li>
-   * <li>
-   * Otherwise, retrieve the mob cell from the mob file, and writes a copy of the cell into
-   * the new store file.
-   * </li>
-   * </ol>
-   * 2. If the cell doesn't have a reference tag.
-   * <ol>
-   * <li>
-   * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
-   * write this cell to a mob file, and write the path of this mob file to the store file.
-   * </li>
-   * <li>
-   * Otherwise, directly write this cell into the store file.
-   * </li>
-   * </ol>
-   * In the mob compaction, the {@link MobCompactionStoreScanner} is used as a scanner
-   * which could output the normal cells and delete markers together when required.
-   * After the major compaction on the normal hfiles, we have a guarantee that we have purged all
-   * deleted or old version mob refs, and the delete markers are written to a del file with the
-   * suffix _del. Because of this, it is safe to use the del file in the mob compaction.
-   * The mob compaction doesn't take place in the normal hfiles, it occurs directly in the
-   * mob files. When the small mob files are merged into bigger ones, the del file is added into
-   * the scanner to filter the deleted cells.
-   * @param fd File details
-   * @param scanner Where to read from.
-   * @param writer Where to write to.
-   * @param smallestReadPoint Smallest read point.
-   * @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
-   * @param major Is a major compaction.
-   * @return Whether compaction ended; false if it was interrupted for any reason.
-   */
-  @Override
-  protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
-      long smallestReadPoint, boolean cleanSeqId,
-      CompactionThroughputController throughputController,  boolean major) throws IOException {
-    if (!(scanner instanceof MobCompactionStoreScanner)) {
-      throw new IllegalArgumentException(
-          "The scanner should be an instance of MobCompactionStoreScanner");
-    }
-    MobCompactionStoreScanner compactionScanner = (MobCompactionStoreScanner) scanner;
-    int bytesWritten = 0;
-    // Since scanner.next() can return 'false' but still be delivering data,
-    // we have to use a do/while loop.
-    List<Cell> cells = new ArrayList<Cell>();
-    // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
-    int closeCheckInterval = HStore.getCloseCheckInterval();
-    boolean hasMore;
-    Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
-    byte[] fileName = null;
-    StoreFile.Writer mobFileWriter = null;
-    StoreFile.Writer delFileWriter = null;
-    long mobCells = 0;
-    long deleteMarkersCount = 0;
-    Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName()
-            .getName());
-    long mobCompactedIntoMobCellsCount = 0;
-    long mobCompactedFromMobCellsCount = 0;
-    long mobCompactedIntoMobCellsSize = 0;
-    long mobCompactedFromMobCellsSize = 0;
-    try {
-      try {
-        // If the mob file writer could not be created, directly write the cell to the store file.
-        mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
-            store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
-        fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
-      } catch (IOException e) {
-        LOG.error(
-            "Fail to create mob writer, "
-                + "we will continue the compaction by writing MOB cells directly in store files",
-            e);
-      }
-      delFileWriter = mobStore.createDelFileWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
-          store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
-      ScannerContext scannerContext =
-              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
-
-
-      do {
-        hasMore = compactionScanner.next(cells, scannerContext);
-        // output to writer:
-        for (Cell c : cells) {
-          if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) {
-            CellUtil.setSequenceId(c, 0);
-          }
-          if (compactionScanner.isOutputDeleteMarkers() && CellUtil.isDelete(c)) {
-            delFileWriter.append(c);
-            deleteMarkersCount++;
-          } else if (mobFileWriter == null || c.getTypeByte() != KeyValue.Type.Put.getCode()) {
-            // If the mob file writer is null or the kv type is not put, directly write the cell
-            // to the store file.
-            writer.append(c);
-          } else if (MobUtils.isMobReferenceCell(c)) {
-            if (MobUtils.hasValidMobRefCellValue(c)) {
-              int size = MobUtils.getMobValueLength(c);
-              if (size > mobSizeThreshold) {
-                // If the value size is larger than the threshold, it's regarded as a mob. Since
-                // its value is already in the mob file, directly write this cell to the store file
-                writer.append(c);
-              } else {
-                // If the value is not larger than the threshold, it's not regarded a mob. Retrieve
-                // the mob cell from the mob file, and write it back to the store file.
-                Cell mobCell = mobStore.resolve(c, false);
-                if (mobCell.getValueLength() != 0) {
-                  // put the mob data back to the store file
-                  CellUtil.setSequenceId(mobCell, c.getSequenceId());
-                  writer.append(mobCell);
-                  mobCompactedFromMobCellsCount++;
-                  mobCompactedFromMobCellsSize += mobCell.getValueLength();
-                } else {
-                  // If the value of a file is empty, there might be issues when retrieving,
-                  // directly write the cell to the store file, and leave it to be handled by the
-                  // next compaction.
-                  writer.append(c);
-                }
-              }
-            } else {
-              LOG.warn("The value format of the KeyValue " + c
-                  + " is wrong, its length is less than " + Bytes.SIZEOF_INT);
-              writer.append(c);
-            }
-          } else if (c.getValueLength() <= mobSizeThreshold) {
-            // If the value size of a cell is not larger than the threshold, directly write it to
-            // the store file.
-            writer.append(c);
-          } else {
-            // If the value size of a cell is larger than the threshold, it's regarded as a mob,
-            // write this cell to a mob file, and write the path to the store file.
-            mobCells++;
-            // append the original keyValue in the mob file.
-            mobFileWriter.append(c);
-            KeyValue reference = MobUtils.createMobRefKeyValue(c, fileName, tableNameTag);
-            // write the cell whose value is the path of a mob file to the store file.
-            writer.append(reference);
-            mobCompactedIntoMobCellsCount++;
-            mobCompactedIntoMobCellsSize += c.getValueLength();
-          }
-          ++progress.currentCompactedKVs;
-
-          // check periodically to see if a system stop is requested
-          if (closeCheckInterval > 0) {
-            bytesWritten += KeyValueUtil.length(c);
-            if (bytesWritten > closeCheckInterval) {
-              bytesWritten = 0;
-              if (!store.areWritesEnabled()) {
-                progress.cancel();
-                return false;
-              }
-            }
-          }
-        }
-        cells.clear();
-      } while (hasMore);
-    } finally {
-      if (mobFileWriter != null) {
-        mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
-        mobFileWriter.close();
-      }
-      if (delFileWriter != null) {
-        delFileWriter.appendMetadata(fd.maxSeqId, major, deleteMarkersCount);
-        delFileWriter.close();
-      }
-    }
-    if (mobFileWriter != null) {
-      if (mobCells > 0) {
-        // If the mob file is not empty, commit it.
-        mobStore.commitFile(mobFileWriter.getPath(), path);
-      } else {
-        try {
-          // If the mob file is empty, delete it instead of committing.
-          store.getFileSystem().delete(mobFileWriter.getPath(), true);
-        } catch (IOException e) {
-          LOG.error("Fail to delete the temp mob file", e);
-        }
-      }
-    }
-    if (delFileWriter != null) {
-      if (deleteMarkersCount > 0) {
-        // If the del file is not empty, commit it.
-        // If the commit fails, the compaction is re-performed again.
-        mobStore.commitFile(delFileWriter.getPath(), path);
-      } else {
-        try {
-          // If the del file is empty, delete it instead of committing.
-          store.getFileSystem().delete(delFileWriter.getPath(), true);
-        } catch (IOException e) {
-          LOG.error("Fail to delete the temp del file", e);
-        }
-      }
-    }
-    mobStore.updateMobCompactedFromMobCellsCount(mobCompactedFromMobCellsCount);
-    mobStore.updateMobCompactedIntoMobCellsCount(mobCompactedIntoMobCellsCount);
-    mobStore.updateMobCompactedFromMobCellsSize(mobCompactedFromMobCellsSize);
-    mobStore.updateMobCompactedIntoMobCellsSize(mobCompactedIntoMobCellsSize);
-    progress.complete();
-    return true;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
new file mode 100644
index 0000000..fbcff85
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -0,0 +1,305 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.TagType;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.*;
+import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
+import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Compact passed set of files in the mob-enabled column family.
+ */
+@InterfaceAudience.Private
+public class DefaultMobStoreCompactor extends DefaultCompactor {
+
+  private static final Log LOG = LogFactory.getLog(DefaultMobStoreCompactor.class);
+  private long mobSizeThreshold;
+  private HMobStore mobStore;
+  public DefaultMobStoreCompactor(Configuration conf, Store store) {
+    super(conf, store);
+    // The mob cells reside in the mob-enabled column family which is held by HMobStore.
+    // During the compaction, the compactor reads the cells from the mob files and
+    // probably creates new mob files. All of these operations are included in HMobStore,
+    // so we need to cast the Store to HMobStore.
+    if (!(store instanceof HMobStore)) {
+      throw new IllegalArgumentException("The store " + store + " is not a HMobStore");
+    }
+    mobStore = (HMobStore) store;
+    mobSizeThreshold = store.getFamily().getMobThreshold();
+  }
+
+  /**
+   * Creates a writer for a new file in a temporary directory.
+   * @param fd The file details.
+   * @param smallestReadPoint The smallest mvcc readPoint across all the scanners in this region.
+   * @return Writer for a new StoreFile in the tmp dir.
+   * @throws IOException
+   */
+  @Override
+  protected Writer createTmpWriter(FileDetails fd, long smallestReadPoint) throws IOException {
+    // make this writer with tags always because of possible new cells with tags.
+    StoreFile.Writer writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression,
+        true, fd.maxMVCCReadpoint >= smallestReadPoint, true);
+    return writer;
+  }
+
+  @Override
+  protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
+      ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException {
+    Scan scan = new Scan();
+    scan.setMaxVersions(store.getFamily().getMaxVersions());
+    if (scanType == ScanType.COMPACT_DROP_DELETES) {
+      // In major compaction, we need to write the delete markers to del files, so we have to
+      // retain the them in scanning.
+      scanType = ScanType.COMPACT_RETAIN_DELETES;
+      return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
+          scanType, smallestReadPoint, earliestPutTs, true);
+    } else {
+      return new MobCompactionStoreScanner(store, store.getScanInfo(), scan, scanners,
+          scanType, smallestReadPoint, earliestPutTs, false);
+    }
+  }
+
+  // TODO refactor to take advantage of the throughput controller.
+
+  /**
+   * Performs compaction on a column family with the mob flag enabled.
+   * This is for when the mob threshold size has changed or if the mob
+   * column family mode has been toggled via an alter table statement.
+   * Compacts the files by the following rules.
+   * 1. If the cell has a mob reference tag, the cell's value is the path of the mob file.
+   * <ol>
+   * <li>
+   * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
+   * directly copy the (with mob tag) cell into the new store file.
+   * </li>
+   * <li>
+   * Otherwise, retrieve the mob cell from the mob file, and writes a copy of the cell into
+   * the new store file.
+   * </li>
+   * </ol>
+   * 2. If the cell doesn't have a reference tag.
+   * <ol>
+   * <li>
+   * If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
+   * write this cell to a mob file, and write the path of this mob file to the store file.
+   * </li>
+   * <li>
+   * Otherwise, directly write this cell into the store file.
+   * </li>
+   * </ol>
+   * In the mob compaction, the {@link MobCompactionStoreScanner} is used as a scanner
+   * which could output the normal cells and delete markers together when required.
+   * After the major compaction on the normal hfiles, we have a guarantee that we have purged all
+   * deleted or old version mob refs, and the delete markers are written to a del file with the
+   * suffix _del. Because of this, it is safe to use the del file in the mob compaction.
+   * The mob compaction doesn't take place in the normal hfiles, it occurs directly in the
+   * mob files. When the small mob files are merged into bigger ones, the del file is added into
+   * the scanner to filter the deleted cells.
+   * @param fd File details
+   * @param scanner Where to read from.
+   * @param writer Where to write to.
+   * @param smallestReadPoint Smallest read point.
+   * @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
+   * @param throughputController The compaction throughput controller.
+   * @param major Is a major compaction.
+   * @return Whether compaction ended; false if it was interrupted for any reason.
+   */
+  @Override
+  protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
+      long smallestReadPoint, boolean cleanSeqId,
+      CompactionThroughputController throughputController,  boolean major) throws IOException {
+    if (!(scanner instanceof MobCompactionStoreScanner)) {
+      throw new IllegalArgumentException(
+          "The scanner should be an instance of MobCompactionStoreScanner");
+    }
+    MobCompactionStoreScanner compactionScanner = (MobCompactionStoreScanner) scanner;
+    int bytesWritten = 0;
+    // Since scanner.next() can return 'false' but still be delivering data,
+    // we have to use a do/while loop.
+    List<Cell> cells = new ArrayList<Cell>();
+    // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
+    int closeCheckInterval = HStore.getCloseCheckInterval();
+    boolean hasMore;
+    Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
+    byte[] fileName = null;
+    StoreFile.Writer mobFileWriter = null;
+    StoreFile.Writer delFileWriter = null;
+    long mobCells = 0;
+    long deleteMarkersCount = 0;
+    Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, store.getTableName()
+            .getName());
+    long cellsCountCompactedToMob = 0;
+    long cellsCountCompactedFromMob = 0;
+    long cellsSizeCompactedToMob = 0;
+    long cellsSizeCompactedFromMob = 0;
+    try {
+      try {
+        // If the mob file writer could not be created, directly write the cell to the store file.
+        mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
+            store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
+        fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
+      } catch (IOException e) {
+        LOG.error(
+            "Failed to create mob writer, "
+                + "we will continue the compaction by writing MOB cells directly in store files",
+            e);
+      }
+      delFileWriter = mobStore.createDelFileWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount,
+          store.getFamily().getCompression(), store.getRegionInfo().getStartKey());
+      ScannerContext scannerContext =
+              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
+      do {
+        hasMore = compactionScanner.next(cells, scannerContext);
+        // output to writer:
+        for (Cell c : cells) {
+          if (cleanSeqId && c.getSequenceId() <= smallestReadPoint) {
+            CellUtil.setSequenceId(c, 0);
+          }
+          if (compactionScanner.isOutputDeleteMarkers() && CellUtil.isDelete(c)) {
+            delFileWriter.append(c);
+            deleteMarkersCount++;
+          } else if (mobFileWriter == null || c.getTypeByte() != KeyValue.Type.Put.getCode()) {
+            // If the mob file writer is null or the kv type is not put, directly write the cell
+            // to the store file.
+            writer.append(c);
+          } else if (MobUtils.isMobReferenceCell(c)) {
+            if (MobUtils.hasValidMobRefCellValue(c)) {
+              int size = MobUtils.getMobValueLength(c);
+              if (size > mobSizeThreshold) {
+                // If the value size is larger than the threshold, it's regarded as a mob. Since
+                // its value is already in the mob file, directly write this cell to the store file
+                writer.append(c);
+              } else {
+                // If the value is not larger than the threshold, it's not regarded a mob. Retrieve
+                // the mob cell from the mob file, and write it back to the store file.
+                Cell mobCell = mobStore.resolve(c, false);
+                if (mobCell.getValueLength() != 0) {
+                  // put the mob data back to the store file
+                  CellUtil.setSequenceId(mobCell, c.getSequenceId());
+                  writer.append(mobCell);
+                  cellsCountCompactedFromMob++;
+                  cellsSizeCompactedFromMob += mobCell.getValueLength();
+                } else {
+                  // If the value of a file is empty, there might be issues when retrieving,
+                  // directly write the cell to the store file, and leave it to be handled by the
+                  // next compaction.
+                  writer.append(c);
+                }
+              }
+            } else {
+              LOG.warn("The value format of the KeyValue " + c
+                  + " is wrong, its length is less than " + Bytes.SIZEOF_INT);
+              writer.append(c);
+            }
+          } else if (c.getValueLength() <= mobSizeThreshold) {
+            // If the value size of a cell is not larger than the threshold, directly write it to
+            // the store file.
+            writer.append(c);
+          } else {
+            // If the value size of a cell is larger than the threshold, it's regarded as a mob,
+            // write this cell to a mob file, and write the path to the store file.
+            mobCells++;
+            // append the original keyValue in the mob file.
+            mobFileWriter.append(c);
+            KeyValue reference = MobUtils.createMobRefKeyValue(c, fileName, tableNameTag);
+            // write the cell whose value is the path of a mob file to the store file.
+            writer.append(reference);
+            cellsCountCompactedToMob++;
+            cellsSizeCompactedToMob += c.getValueLength();
+          }
+          ++progress.currentCompactedKVs;
+
+          // check periodically to see if a system stop is requested
+          if (closeCheckInterval > 0) {
+            bytesWritten += KeyValueUtil.length(c);
+            if (bytesWritten > closeCheckInterval) {
+              bytesWritten = 0;
+              if (!store.areWritesEnabled()) {
+                progress.cancel();
+                return false;
+              }
+            }
+          }
+        }
+        cells.clear();
+      } while (hasMore);
+    } finally {
+      if (mobFileWriter != null) {
+        mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
+        mobFileWriter.close();
+      }
+      if (delFileWriter != null) {
+        delFileWriter.appendMetadata(fd.maxSeqId, major, deleteMarkersCount);
+        delFileWriter.close();
+      }
+    }
+    if (mobFileWriter != null) {
+      if (mobCells > 0) {
+        // If the mob file is not empty, commit it.
+        mobStore.commitFile(mobFileWriter.getPath(), path);
+      } else {
+        try {
+          // If the mob file is empty, delete it instead of committing.
+          store.getFileSystem().delete(mobFileWriter.getPath(), true);
+        } catch (IOException e) {
+          LOG.error("Failed to delete the temp mob file", e);
+        }
+      }
+    }
+    if (delFileWriter != null) {
+      if (deleteMarkersCount > 0) {
+        // If the del file is not empty, commit it.
+        // If the commit fails, the compaction is re-performed again.
+        mobStore.commitFile(delFileWriter.getPath(), path);
+      } else {
+        try {
+          // If the del file is empty, delete it instead of committing.
+          store.getFileSystem().delete(delFileWriter.getPath(), true);
+        } catch (IOException e) {
+          LOG.error("Failed to delete the temp del file", e);
+        }
+      }
+    }
+    mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
+    mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
+    mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
+    mobStore.updateCellsSizeCompactedToMob(cellsSizeCompactedToMob);
+    progress.complete();
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 608f4e2..47a0acf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
@@ -121,7 +121,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
     } finally {
       scanner.close();
     }
-    LOG.info("Flushed, sequenceid=" + cacheFlushId + ", memsize="
+    LOG.info("Mob store is flushed, sequenceid=" + cacheFlushId + ", memsize="
         + StringUtils.TraditionalBinaryPrefix.long2String(snapshot.getSize(), "", 1) +
         ", hasBloomFilter=" + writer.hasGeneralBloom() +
         ", into tmp file " + writer.getPath());
@@ -213,7 +213,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
         // If the mob file is empty, delete it instead of committing.
         store.getFileSystem().delete(mobFileWriter.getPath(), true);
       } catch (IOException e) {
-        LOG.error("Fail to delete the temp mob file", e);
+        LOG.error("Failed to delete the temp mob file", e);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
index 7f38c44..703ebd6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java
@@ -112,7 +112,7 @@ public class ExpiredMobFileCleaner extends Configured implements Tool {
       try {
         admin.close();
       } catch (IOException e) {
-        LOG.error("Fail to close the HBaseAdmin.", e);
+        LOG.error("Failed to close the HBaseAdmin.", e);
       }
     }
   }

[4/6] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)

[an...@apache.org]

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactor.java
deleted file mode 100644
index b3c7d83..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/filecompactions/PartitionedMobFileCompactor.java
+++ /dev/null
@@ -1,636 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.crypto.Encryption;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobFileName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactionRequest.CompactionType;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartition;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartitionId;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.regionserver.StoreFile.Writer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * An implementation of {@link MobFileCompactor} that compacts the mob files in partitions.
- */
-@InterfaceAudience.Private
-public class PartitionedMobFileCompactor extends MobFileCompactor {
-
-  private static final Log LOG = LogFactory.getLog(PartitionedMobFileCompactor.class);
-  protected long mergeableSize;
-  protected int delFileMaxCount;
-  /** The number of files compacted in a batch */
-  protected int compactionBatchSize;
-  protected int compactionKVMax;
-
-  private Path tempPath;
-  private Path bulkloadPath;
-  private CacheConfig compactionCacheConfig;
-  private Tag tableNameTag;
-  private Encryption.Context cryptoContext = Encryption.Context.NONE;
-
-  public PartitionedMobFileCompactor(Configuration conf, FileSystem fs, TableName tableName,
-    HColumnDescriptor column, ExecutorService pool) throws IOException {
-    super(conf, fs, tableName, column, pool);
-    mergeableSize = conf.getLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD);
-    delFileMaxCount = conf.getInt(MobConstants.MOB_DELFILE_MAX_COUNT,
-      MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
-    // default is 100
-    compactionBatchSize = conf.getInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE);
-    tempPath = new Path(MobUtils.getMobHome(conf), MobConstants.TEMP_DIR_NAME);
-    bulkloadPath = new Path(tempPath, new Path(MobConstants.BULKLOAD_DIR_NAME, new Path(
-      tableName.getNamespaceAsString(), tableName.getQualifierAsString())));
-    compactionKVMax = this.conf.getInt(HConstants.COMPACTION_KV_MAX,
-      HConstants.COMPACTION_KV_MAX_DEFAULT);
-    Configuration copyOfConf = new Configuration(conf);
-    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
-    compactionCacheConfig = new CacheConfig(copyOfConf);
-    tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, tableName.getName());
-    cryptoContext = MobUtils.createEncryptionContext(copyOfConf, column);
-  }
-
-  @Override
-  public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles) throws IOException {
-    if (files == null || files.isEmpty()) {
-      LOG.info("No candidate mob files");
-      return null;
-    }
-    LOG.info("isForceAllFiles: " + isForceAllFiles);
-    // find the files to compact.
-    PartitionedMobFileCompactionRequest request = select(files, isForceAllFiles);
-    // compact the files.
-    return performCompaction(request);
-  }
-
-  /**
-   * Selects the compacted mob/del files.
-   * Iterates the candidates to find out all the del files and small mob files.
-   * @param candidates All the candidates.
-   * @param isForceAllFiles Whether add all mob files into the compaction.
-   * @return A compaction request.
-   * @throws IOException
-   */
-  protected PartitionedMobFileCompactionRequest select(List<FileStatus> candidates,
-    boolean isForceAllFiles) throws IOException {
-    Collection<FileStatus> allDelFiles = new ArrayList<FileStatus>();
-    Map<CompactionPartitionId, CompactionPartition> filesToCompact =
-      new HashMap<CompactionPartitionId, CompactionPartition>();
-    int selectedFileCount = 0;
-    int irrelevantFileCount = 0;
-    for (FileStatus file : candidates) {
-      if (!file.isFile()) {
-        irrelevantFileCount++;
-        continue;
-      }
-      // group the del files and small files.
-      FileStatus linkedFile = file;
-      if (HFileLink.isHFileLink(file.getPath())) {
-        HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, file.getPath());
-        linkedFile = getLinkedFileStatus(link);
-        if (linkedFile == null) {
-          // If the linked file cannot be found, regard it as an irrelevantFileCount file
-          irrelevantFileCount++;
-          continue;
-        }
-      }
-      if (StoreFileInfo.isDelFile(linkedFile.getPath())) {
-        allDelFiles.add(file);
-      } else if (isForceAllFiles || linkedFile.getLen() < mergeableSize) {
-        // add all files if isForceAllFiles is true,
-        // otherwise add the small files to the merge pool
-        MobFileName fileName = MobFileName.create(linkedFile.getPath().getName());
-        CompactionPartitionId id = new CompactionPartitionId(fileName.getStartKey(),
-          fileName.getDate());
-        CompactionPartition compactionPartition = filesToCompact.get(id);
-        if (compactionPartition == null) {
-          compactionPartition = new CompactionPartition(id);
-          compactionPartition.addFile(file);
-          filesToCompact.put(id, compactionPartition);
-        } else {
-          compactionPartition.addFile(file);
-        }
-        selectedFileCount++;
-      }
-    }
-    PartitionedMobFileCompactionRequest request = new PartitionedMobFileCompactionRequest(
-      filesToCompact.values(), allDelFiles);
-    if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount)) {
-      // all the files are selected
-      request.setCompactionType(CompactionType.ALL_FILES);
-    }
-    LOG.info("The compaction type is " + request.getCompactionType() + ", the request has "
-      + allDelFiles.size() + " del files, " + selectedFileCount + " selected files, and "
-      + irrelevantFileCount + " irrelevant files");
-    return request;
-  }
-
-  /**
-   * Performs the compaction on the selected files.
-   * <ol>
-   * <li>Compacts the del files.</li>
-   * <li>Compacts the selected small mob files and all the del files.</li>
-   * <li>If all the candidates are selected, delete the del files.</li>
-   * </ol>
-   * @param request The compaction request.
-   * @return The paths of new mob files generated in the compaction.
-   * @throws IOException
-   */
-  protected List<Path> performCompaction(PartitionedMobFileCompactionRequest request)
-    throws IOException {
-    // merge the del files
-    List<Path> delFilePaths = new ArrayList<Path>();
-    for (FileStatus delFile : request.delFiles) {
-      delFilePaths.add(delFile.getPath());
-    }
-    List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
-    List<StoreFile> newDelFiles = new ArrayList<StoreFile>();
-    for (Path newDelPath : newDelPaths) {
-      StoreFile sf = new StoreFile(fs, newDelPath, conf, compactionCacheConfig, BloomType.NONE);
-      newDelFiles.add(sf);
-    }
-    LOG.info("After merging, there are " + newDelFiles.size() + " del files");
-    // compact the mob files by partitions.
-    List<Path> paths = compactMobFiles(request, newDelFiles);
-    LOG.info("After compaction, there are " + paths.size() + " mob files");
-    // archive the del files if all the mob files are selected.
-    if (request.type == CompactionType.ALL_FILES && !newDelPaths.isEmpty()) {
-      LOG.info("After a mob file compaction with all files selected, archiving the del files "
-        + newDelFiles);
-      try {
-        MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), newDelFiles);
-      } catch (IOException e) {
-        LOG.error("Failed to archive the del files " + newDelFiles, e);
-      }
-    }
-    return paths;
-  }
-
-  /**
-   * Compacts the selected small mob files and all the del files.
-   * @param request The compaction request.
-   * @param delFiles The del files.
-   * @return The paths of new mob files after compactions.
-   * @throws IOException
-   */
-  protected List<Path> compactMobFiles(final PartitionedMobFileCompactionRequest request,
-    final List<StoreFile> delFiles) throws IOException {
-    Collection<CompactionPartition> partitions = request.compactionPartitions;
-    if (partitions == null || partitions.isEmpty()) {
-      LOG.info("No partitions of mob files");
-      return Collections.emptyList();
-    }
-    List<Path> paths = new ArrayList<Path>();
-    Connection c = ConnectionFactory.createConnection(conf);
-    final Table table = c.getTable(tableName);
-    try {
-      Map<CompactionPartitionId, Future<List<Path>>> results =
-        new HashMap<CompactionPartitionId, Future<List<Path>>>();
-      // compact the mob files by partitions in parallel.
-      for (final CompactionPartition partition : partitions) {
-        results.put(partition.getPartitionId(), pool.submit(new Callable<List<Path>>() {
-          @Override
-          public List<Path> call() throws Exception {
-            LOG.info("Compacting mob files for partition " + partition.getPartitionId());
-            return compactMobFilePartition(request, partition, delFiles, table);
-          }
-        }));
-      }
-      // compact the partitions in parallel.
-      boolean hasFailure = false;
-      for (Entry<CompactionPartitionId, Future<List<Path>>> result : results.entrySet()) {
-        try {
-          paths.addAll(result.getValue().get());
-        } catch (Exception e) {
-          // just log the error
-          LOG.error("Failed to compact the partition " + result.getKey(), e);
-          hasFailure = true;
-        }
-      }
-      if (hasFailure) {
-        // if any partition fails in the compaction, directly throw an exception.
-        throw new IOException("Failed to compact the partitions");
-      }
-    } finally {
-      try {
-        table.close();
-      } catch (IOException e) {
-        LOG.error("Failed to close the HTable", e);
-      }
-    }
-    return paths;
-  }
-
-  /**
-   * Compacts a partition of selected small mob files and all the del files.
-   * @param request The compaction request.
-   * @param partition A compaction partition.
-   * @param delFiles The del files.
-   * @param table The current table.
-   * @return The paths of new mob files after compactions.
-   * @throws IOException
-   */
-  private List<Path> compactMobFilePartition(PartitionedMobFileCompactionRequest request,
-    CompactionPartition partition, List<StoreFile> delFiles, Table table) throws IOException {
-    List<Path> newFiles = new ArrayList<Path>();
-    List<FileStatus> files = partition.listFiles();
-    int offset = 0;
-    Path bulkloadPathOfPartition = new Path(bulkloadPath, partition.getPartitionId().toString());
-    Path bulkloadColumnPath = new Path(bulkloadPathOfPartition, column.getNameAsString());
-    while (offset < files.size()) {
-      int batch = compactionBatchSize;
-      if (files.size() - offset < compactionBatchSize) {
-        batch = files.size() - offset;
-      }
-      if (batch == 1 && delFiles.isEmpty()) {
-        // only one file left and no del files, do not compact it,
-        // and directly add it to the new files.
-        newFiles.add(files.get(offset).getPath());
-        offset++;
-        continue;
-      }
-      // clean the bulkload directory to avoid loading old files.
-      fs.delete(bulkloadPathOfPartition, true);
-      // add the selected mob files and del files into filesToCompact
-      List<StoreFile> filesToCompact = new ArrayList<StoreFile>();
-      for (int i = offset; i < batch + offset; i++) {
-        StoreFile sf = new StoreFile(fs, files.get(i).getPath(), conf, compactionCacheConfig,
-          BloomType.NONE);
-        filesToCompact.add(sf);
-      }
-      filesToCompact.addAll(delFiles);
-      // compact the mob files in a batch.
-      compactMobFilesInBatch(request, partition, table, filesToCompact, batch,
-        bulkloadPathOfPartition, bulkloadColumnPath, newFiles);
-      // move to the next batch.
-      offset += batch;
-    }
-    LOG.info("Compaction is finished. The number of mob files is changed from " + files.size()
-      + " to " + newFiles.size());
-    return newFiles;
-  }
-
-  /**
-   * Compacts a partition of selected small mob files and all the del files in a batch.
-   * @param request The compaction request.
-   * @param partition A compaction partition.
-   * @param table The current table.
-   * @param filesToCompact The files to be compacted.
-   * @param batch The number of mob files to be compacted in a batch.
-   * @param bulkloadPathOfPartition The directory where the bulkload column of the current
-   *        partition is saved.
-   * @param bulkloadColumnPath The directory where the bulkload files of current partition
-   *        are saved.
-   * @param newFiles The paths of new mob files after compactions.
-   * @throws IOException
-   */
-  private void compactMobFilesInBatch(PartitionedMobFileCompactionRequest request,
-    CompactionPartition partition, Table table, List<StoreFile> filesToCompact, int batch,
-    Path bulkloadPathOfPartition, Path bulkloadColumnPath, List<Path> newFiles)
-    throws IOException {
-    // open scanner to the selected mob files and del files.
-    StoreScanner scanner = createScanner(filesToCompact, ScanType.COMPACT_DROP_DELETES);
-    // the mob files to be compacted, not include the del files.
-    List<StoreFile> mobFilesToCompact = filesToCompact.subList(0, batch);
-    // Pair(maxSeqId, cellsCount)
-    Pair<Long, Long> fileInfo = getFileInfo(mobFilesToCompact);
-    // open writers for the mob files and new ref store files.
-    Writer writer = null;
-    Writer refFileWriter = null;
-    Path filePath = null;
-    Path refFilePath = null;
-    long mobCells = 0;
-    try {
-      writer = MobUtils.createWriter(conf, fs, column, partition.getPartitionId().getDate(),
-        tempPath, Long.MAX_VALUE, column.getCompactionCompression(), partition.getPartitionId()
-          .getStartKey(), compactionCacheConfig, cryptoContext);
-      filePath = writer.getPath();
-      byte[] fileName = Bytes.toBytes(filePath.getName());
-      // create a temp file and open a writer for it in the bulkloadPath
-      refFileWriter = MobUtils.createRefFileWriter(conf, fs, column, bulkloadColumnPath, fileInfo
-        .getSecond().longValue(), compactionCacheConfig, cryptoContext);
-      refFilePath = refFileWriter.getPath();
-      List<Cell> cells = new ArrayList<Cell>();
-      boolean hasMore = false;
-      ScannerContext scannerContext =
-              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
-      do {
-        hasMore = scanner.next(cells, scannerContext);
-        for (Cell cell : cells) {
-          // write the mob cell to the mob file.
-          writer.append(cell);
-          // write the new reference cell to the store file.
-          KeyValue reference = MobUtils.createMobRefKeyValue(cell, fileName, tableNameTag);
-          refFileWriter.append(reference);
-          mobCells++;
-        }
-        cells.clear();
-      } while (hasMore);
-    } finally {
-      // close the scanner.
-      scanner.close();
-      // append metadata to the mob file, and close the mob file writer.
-      closeMobFileWriter(writer, fileInfo.getFirst(), mobCells);
-      // append metadata and bulkload info to the ref mob file, and close the writer.
-      closeRefFileWriter(refFileWriter, fileInfo.getFirst(), request.selectionTime);
-    }
-    if (mobCells > 0) {
-      // commit mob file
-      MobUtils.commitFile(conf, fs, filePath, mobFamilyDir, compactionCacheConfig);
-      // bulkload the ref file
-      bulkloadRefFile(table, bulkloadPathOfPartition, filePath.getName());
-      newFiles.add(new Path(mobFamilyDir, filePath.getName()));
-    } else {
-      // remove the new files
-      // the mob file is empty, delete it instead of committing.
-      deletePath(filePath);
-      // the ref file is empty, delete it instead of committing.
-      deletePath(refFilePath);
-    }
-    // archive the old mob files, do not archive the del files.
-    try {
-      MobUtils
-        .removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), mobFilesToCompact);
-    } catch (IOException e) {
-      LOG.error("Failed to archive the files " + mobFilesToCompact, e);
-    }
-  }
-
-  /**
-   * Compacts the del files in batches which avoids opening too many files.
-   * @param request The compaction request.
-   * @param delFilePaths
-   * @return The paths of new del files after merging or the original files if no merging
-   *         is necessary.
-   * @throws IOException
-   */
-  protected List<Path> compactDelFiles(PartitionedMobFileCompactionRequest request,
-    List<Path> delFilePaths) throws IOException {
-    if (delFilePaths.size() <= delFileMaxCount) {
-      return delFilePaths;
-    }
-    // when there are more del files than the number that is allowed, merge it firstly.
-    int offset = 0;
-    List<Path> paths = new ArrayList<Path>();
-    while (offset < delFilePaths.size()) {
-      // get the batch
-      int batch = compactionBatchSize;
-      if (delFilePaths.size() - offset < compactionBatchSize) {
-        batch = delFilePaths.size() - offset;
-      }
-      List<StoreFile> batchedDelFiles = new ArrayList<StoreFile>();
-      if (batch == 1) {
-        // only one file left, do not compact it, directly add it to the new files.
-        paths.add(delFilePaths.get(offset));
-        offset++;
-        continue;
-      }
-      for (int i = offset; i < batch + offset; i++) {
-        batchedDelFiles.add(new StoreFile(fs, delFilePaths.get(i), conf, compactionCacheConfig,
-          BloomType.NONE));
-      }
-      // compact the del files in a batch.
-      paths.add(compactDelFilesInBatch(request, batchedDelFiles));
-      // move to the next batch.
-      offset += batch;
-    }
-    return compactDelFiles(request, paths);
-  }
-
-  /**
-   * Compacts the del file in a batch.
-   * @param request The compaction request.
-   * @param delFiles The del files.
-   * @return The path of new del file after merging.
-   * @throws IOException
-   */
-  private Path compactDelFilesInBatch(PartitionedMobFileCompactionRequest request,
-    List<StoreFile> delFiles) throws IOException {
-    // create a scanner for the del files.
-    StoreScanner scanner = createScanner(delFiles, ScanType.COMPACT_RETAIN_DELETES);
-    Writer writer = null;
-    Path filePath = null;
-    try {
-      writer = MobUtils.createDelFileWriter(conf, fs, column,
-        MobUtils.formatDate(new Date(request.selectionTime)), tempPath, Long.MAX_VALUE,
-        column.getCompactionCompression(), HConstants.EMPTY_START_ROW, compactionCacheConfig,
-        cryptoContext);
-      filePath = writer.getPath();
-      List<Cell> cells = new ArrayList<Cell>();
-      boolean hasMore = false;
-      ScannerContext scannerContext =
-              ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
-      do {
-        hasMore = scanner.next(cells, scannerContext);
-        for (Cell cell : cells) {
-          writer.append(cell);
-        }
-        cells.clear();
-      } while (hasMore);
-    } finally {
-      scanner.close();
-      if (writer != null) {
-        try {
-          writer.close();
-        } catch (IOException e) {
-          LOG.error("Failed to close the writer of the file " + filePath, e);
-        }
-      }
-    }
-    // commit the new del file
-    Path path = MobUtils.commitFile(conf, fs, filePath, mobFamilyDir, compactionCacheConfig);
-    // archive the old del files
-    try {
-      MobUtils.removeMobFiles(conf, fs, tableName, mobTableDir, column.getName(), delFiles);
-    } catch (IOException e) {
-      LOG.error("Failed to archive the old del files " + delFiles, e);
-    }
-    return path;
-  }
-
-  /**
-   * Creates a store scanner.
-   * @param filesToCompact The files to be compacted.
-   * @param scanType The scan type.
-   * @return The store scanner.
-   * @throws IOException
-   */
-  private StoreScanner createScanner(List<StoreFile> filesToCompact, ScanType scanType)
-    throws IOException {
-    List scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, true, false,
-      null, HConstants.LATEST_TIMESTAMP);
-    Scan scan = new Scan();
-    scan.setMaxVersions(column.getMaxVersions());
-    long ttl = HStore.determineTTLFromFamily(column);
-    ScanInfo scanInfo = new ScanInfo(column, ttl, 0, CellComparator.COMPARATOR);
-    StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners, 0L,
-      HConstants.LATEST_TIMESTAMP);
-    return scanner;
-  }
-
-  /**
-   * Bulkloads the current file.
-   * @param table The current table.
-   * @param bulkloadDirectory The path of bulkload directory.
-   * @param fileName The current file name.
-   * @throws IOException
-   */
-  private void bulkloadRefFile(Table table, Path bulkloadDirectory, String fileName)
-    throws IOException {
-    // bulkload the ref file
-    try {
-      LoadIncrementalHFiles bulkload = new LoadIncrementalHFiles(conf);
-      bulkload.doBulkLoad(bulkloadDirectory, (HTable)table);
-    } catch (Exception e) {
-      // delete the committed mob file
-      deletePath(new Path(mobFamilyDir, fileName));
-      throw new IOException(e);
-    } finally {
-      // delete the bulkload files in bulkloadPath
-      deletePath(bulkloadDirectory);
-    }
-  }
-
-  /**
-   * Closes the mob file writer.
-   * @param writer The mob file writer.
-   * @param maxSeqId Maximum sequence id.
-   * @param mobCellsCount The number of mob cells.
-   * @throws IOException
-   */
-  private void closeMobFileWriter(Writer writer, long maxSeqId, long mobCellsCount)
-    throws IOException {
-    if (writer != null) {
-      writer.appendMetadata(maxSeqId, false, mobCellsCount);
-      try {
-        writer.close();
-      } catch (IOException e) {
-        LOG.error("Failed to close the writer of the file " + writer.getPath(), e);
-      }
-    }
-  }
-
-  /**
-   * Closes the ref file writer.
-   * @param writer The ref file writer.
-   * @param maxSeqId Maximum sequence id.
-   * @param bulkloadTime The timestamp at which the bulk load file is created.
-   * @throws IOException
-   */
-  private void closeRefFileWriter(Writer writer, long maxSeqId, long bulkloadTime)
-    throws IOException {
-    if (writer != null) {
-      writer.appendMetadata(maxSeqId, false);
-      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(bulkloadTime));
-      try {
-        writer.close();
-      } catch (IOException e) {
-        LOG.error("Failed to close the writer of the ref file " + writer.getPath(), e);
-      }
-    }
-  }
-
-  /**
-   * Gets the max seqId and number of cells of the store files.
-   * @param storeFiles The store files.
-   * @return The pair of the max seqId and number of cells of the store files.
-   * @throws IOException
-   */
-  private Pair<Long, Long> getFileInfo(List<StoreFile> storeFiles) throws IOException {
-    long maxSeqId = 0;
-    long maxKeyCount = 0;
-    for (StoreFile sf : storeFiles) {
-      // the readers will be closed later after the merge.
-      maxSeqId = Math.max(maxSeqId, sf.getMaxSequenceId());
-      byte[] count = sf.createReader().loadFileInfo().get(StoreFile.MOB_CELLS_COUNT);
-      if (count != null) {
-        maxKeyCount += Bytes.toLong(count);
-      }
-    }
-    return new Pair<Long, Long>(Long.valueOf(maxSeqId), Long.valueOf(maxKeyCount));
-  }
-
-  /**
-   * Deletes a file.
-   * @param path The path of the file to be deleted.
-   */
-  private void deletePath(Path path) {
-    try {
-      if (path != null) {
-        fs.delete(path, true);
-      }
-    } catch (IOException e) {
-      LOG.error("Failed to delete the file " + path, e);
-    }
-  }
-
-  private FileStatus getLinkedFileStatus(HFileLink link) throws IOException {
-    Path[] locations = link.getLocations();
-    for (Path location : locations) {
-      FileStatus file = getFileStatus(location);
-      if (file != null) {
-        return file;
-      }
-    }
-    return null;
-  }
-
-  private FileStatus getFileStatus(Path path) throws IOException {
-    try {
-      if (path != null) {
-        FileStatus file = fs.getFileStatus(path);
-        return file;
-      }
-    } catch (FileNotFoundException e) {
-      LOG.warn("The file " + path + " can not be found", e);
-    }
-    return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
index 458e187..82d03cd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MemStoreWrapper.java
@@ -39,8 +39,8 @@ import org.apache.hadoop.hbase.io.crypto.Encryption;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartitionId;
 import org.apache.hadoop.hbase.mob.mapreduce.SweepJob.SweepCounter;
-import org.apache.hadoop.hbase.mob.mapreduce.SweepReducer.SweepPartitionId;
 import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.MemStore;
 import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
@@ -68,7 +68,7 @@ public class MemStoreWrapper {
 
   private MemStore memstore;
   private long flushSize;
-  private SweepPartitionId partitionId;
+  private CompactionPartitionId partitionId;
   private Context context;
   private Configuration conf;
   private BufferedMutator table;
@@ -78,8 +78,8 @@ public class MemStoreWrapper {
   private CacheConfig cacheConfig;
   private Encryption.Context cryptoContext = Encryption.Context.NONE;
 
-  public MemStoreWrapper(Context context, FileSystem fs, BufferedMutator table, HColumnDescriptor hcd,
-      MemStore memstore, CacheConfig cacheConfig) throws IOException {
+  public MemStoreWrapper(Context context, FileSystem fs, BufferedMutator table,
+    HColumnDescriptor hcd, MemStore memstore, CacheConfig cacheConfig) throws IOException {
     this.memstore = memstore;
     this.context = context;
     this.fs = fs;
@@ -93,7 +93,7 @@ public class MemStoreWrapper {
     cryptoContext = MobUtils.createEncryptionContext(conf, hcd);
   }
 
-  public void setPartitionId(SweepPartitionId partitionId) {
+  public void setPartitionId(CompactionPartitionId partitionId) {
     this.partitionId = partitionId;
   }
 
@@ -155,16 +155,19 @@ public class MemStoreWrapper {
     scanner = snapshot.getScanner();
     scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
     cell = null;
-    Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
+    Tag tableNameTag = new Tag(TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName()
+      .toString()));
+    long updatedCount = 0;
     while (null != (cell = scanner.next())) {
       KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
       Put put =
           new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
       put.add(reference);
       table.mutate(put);
-      context.getCounter(SweepCounter.RECORDS_UPDATED).increment(1);
+      updatedCount++;
     }
     table.flush();
+    context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
     scanner.close();
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
index 6e4ea98..5da220f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepJob.java
@@ -85,12 +85,12 @@ public class SweepJob {
   private final FileSystem fs;
   private final Configuration conf;
   private static final Log LOG = LogFactory.getLog(SweepJob.class);
-  static final String SWEEP_JOB_ID = "mob.sweep.job.id";
-  static final String SWEEP_JOB_SERVERNAME = "mob.sweep.job.servername";
-  static final String SWEEP_JOB_TABLE_NODE = "mob.sweep.job.table.node";
-  static final String WORKING_DIR_KEY = "mob.sweep.job.dir";
-  static final String WORKING_ALLNAMES_FILE_KEY = "mob.sweep.job.all.file";
-  static final String WORKING_VISITED_DIR_KEY = "mob.sweep.job.visited.dir";
+  static final String SWEEP_JOB_ID = "hbase.mob.sweep.job.id";
+  static final String SWEEP_JOB_SERVERNAME = "hbase.mob.sweep.job.servername";
+  static final String SWEEP_JOB_TABLE_NODE = "hbase.mob.sweep.job.table.node";
+  static final String WORKING_DIR_KEY = "hbase.mob.sweep.job.dir";
+  static final String WORKING_ALLNAMES_FILE_KEY = "hbase.mob.sweep.job.all.file";
+  static final String WORKING_VISITED_DIR_KEY = "hbase.mob.sweep.job.visited.dir";
   static final String WORKING_ALLNAMES_DIR = "all";
   static final String WORKING_VISITED_DIR = "visited";
   public static final String WORKING_FILES_DIR_KEY = "mob.sweep.job.files.dir";
@@ -228,7 +228,7 @@ public class SweepJob {
           try {
             lock.release();
           } catch (IOException e) {
-            LOG.error("Fail to release the table lock " + tableName, e);
+            LOG.error("Failed to release the table lock " + tableName, e);
           }
         }
       }
@@ -435,7 +435,7 @@ public class SweepJob {
             FSUtils.getTableDir(MobUtils.getMobHome(conf), tn), hcd.getName(), storeFiles);
         LOG.info(storeFiles.size() + " unused MOB files are removed");
       } catch (Exception e) {
-        LOG.error("Fail to archive the store files " + storeFiles, e);
+        LOG.error("Failed to archive the store files " + storeFiles, e);
       }
     }
   }
@@ -452,7 +452,7 @@ public class SweepJob {
       try {
         fs.delete(workingPath, true);
       } catch (IOException e) {
-        LOG.warn("Fail to delete the working directory after sweeping store " + familyName
+        LOG.warn("Failed to delete the working directory after sweeping store " + familyName
             + " in the table " + tn.getNameAsString(), e);
       }
     }
@@ -480,10 +480,12 @@ public class SweepJob {
 
     @Override
     public int compareTo(IndexedResult o) {
-      if (this.value == null) {
+      if (this.value == null && o.getValue() == null) {
         return 0;
       } else if (o.value == null) {
         return 1;
+      } else if (this.value == null) {
+        return -1;
       } else {
         return this.value.compareTo(o.value);
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java
index 787b242..a2dfa29 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/SweepReducer.java
@@ -43,7 +43,11 @@ import org.apache.hadoop.hbase.InvalidFamilyOperationException;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.BufferedMutatorParams;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.io.HFileLink;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
@@ -51,6 +55,7 @@ import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobFile;
 import org.apache.hadoop.hbase.mob.MobFileName;
 import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactionRequest.CompactionPartitionId;
 import org.apache.hadoop.hbase.mob.mapreduce.SweepJob.DummyMobAbortable;
 import org.apache.hadoop.hbase.mob.mapreduce.SweepJob.SweepCounter;
 import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -120,7 +125,7 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
       try {
         admin.close();
       } catch (IOException e) {
-        LOG.warn("Fail to close the HBaseAdmin", e);
+        LOG.warn("Failed to close the HBaseAdmin", e);
       }
     }
     // disable the block cache.
@@ -138,7 +143,8 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
     mobTableDir = FSUtils.getTableDir(MobUtils.getMobHome(conf), tn);
   }
 
-  private SweepPartition createPartition(SweepPartitionId id, Context context) throws IOException {
+  private SweepPartition createPartition(CompactionPartitionId id, Context context)
+    throws IOException {
     return new SweepPartition(id, context);
   }
 
@@ -161,13 +167,13 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
       fout = fs.create(nameFilePath, true);
       writer = SequenceFile.createWriter(context.getConfiguration(), fout, String.class,
           String.class, CompressionType.NONE, null);
-      SweepPartitionId id;
+      CompactionPartitionId id;
       SweepPartition partition = null;
       // the mob files which have the same start key and date are in the same partition.
       while (context.nextKey()) {
         Text key = context.getCurrentKey();
         String keyString = key.toString();
-        id = SweepPartitionId.create(keyString);
+        id = createPartitionId(keyString);
         if (null == partition || !id.equals(partition.getId())) {
           // It's the first mob file in the current partition.
           if (null != partition) {
@@ -215,21 +221,21 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
    */
   public class SweepPartition {
 
-    private final SweepPartitionId id;
+    private final CompactionPartitionId id;
     private final Context context;
     private boolean memstoreUpdated = false;
     private boolean mergeSmall = false;
     private final Map<String, MobFileStatus> fileStatusMap = new HashMap<String, MobFileStatus>();
     private final List<Path> toBeDeleted = new ArrayList<Path>();
 
-    public SweepPartition(SweepPartitionId id, Context context) throws IOException {
+    public SweepPartition(CompactionPartitionId id, Context context) throws IOException {
       this.id = id;
       this.context = context;
       memstore.setPartitionId(id);
       init();
     }
 
-    public SweepPartitionId getId() {
+    public CompactionPartitionId getId() {
       return this.id;
     }
 
@@ -294,7 +300,7 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
               storeFiles);
           context.getCounter(SweepCounter.FILE_TO_BE_MERGE_OR_CLEAN).increment(storeFiles.size());
         } catch (IOException e) {
-          LOG.error("Fail to archive the store files " + storeFiles, e);
+          LOG.error("Failed to archive the store files " + storeFiles, e);
         }
         storeFiles.clear();
       }
@@ -390,58 +396,13 @@ public class SweepReducer extends Reducer<Text, KeyValue, Writable, Writable> {
   }
 
   /**
-   * The sweep partition id.
-   * It consists of the start key and date.
-   * The start key is a hex string of the checksum of a region start key.
-   * The date is the latest timestamp of cells in a mob file.
+   * Creates the partition id.
+   * @param fileNameAsString The current file name, in string.
+   * @return The partition id.
    */
-  public static class SweepPartitionId {
-    private String date;
-    private String startKey;
-
-    public SweepPartitionId(MobFileName fileName) {
-      this.date = fileName.getDate();
-      this.startKey = fileName.getStartKey();
-    }
-
-    public SweepPartitionId(String date, String startKey) {
-      this.date = date;
-      this.startKey = startKey;
-    }
-
-    public static SweepPartitionId create(String key) {
-      return new SweepPartitionId(MobFileName.create(key));
-    }
-
-    @Override
-    public boolean equals(Object anObject) {
-      if (this == anObject) {
-        return true;
-      }
-      if (anObject instanceof SweepPartitionId) {
-        SweepPartitionId another = (SweepPartitionId) anObject;
-        if (this.date.equals(another.getDate()) && this.startKey.equals(another.getStartKey())) {
-          return true;
-        }
-      }
-      return false;
-    }
-
-    public String getDate() {
-      return this.date;
-    }
-
-    public String getStartKey() {
-      return this.startKey;
-    }
-
-    public void setDate(String date) {
-      this.date = date;
-    }
-
-    public void setStartKey(String startKey) {
-      this.startKey = startKey;
-    }
+  private CompactionPartitionId createPartitionId(String fileNameAsString) {
+    MobFileName fileName = MobFileName.create(fileNameAsString);
+    return new CompactionPartitionId(fileName.getStartKey(), fileName.getDate());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/Sweeper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/Sweeper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/Sweeper.java
index 9342a31..5436554 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/Sweeper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/Sweeper.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.util.Tool;
@@ -43,6 +44,7 @@ import com.google.protobuf.ServiceException;
  * same column family are mutually exclusive too.
  */
 @InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class Sweeper extends Configured implements Tool {
 
   /**
@@ -82,7 +84,7 @@ public class Sweeper extends Configured implements Tool {
       try {
         admin.close();
       } catch (IOException e) {
-        System.out.println("Fail to close the HBaseAdmin: " + e.getMessage());
+        System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
index dd5d895..4a782dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java
@@ -85,10 +85,10 @@ public class HMobStore extends HStore {
   private MobCacheConfig mobCacheConfig;
   private Path homePath;
   private Path mobFamilyPath;
-  private volatile long mobCompactedIntoMobCellsCount = 0;
-  private volatile long mobCompactedFromMobCellsCount = 0;
-  private volatile long mobCompactedIntoMobCellsSize = 0;
-  private volatile long mobCompactedFromMobCellsSize = 0;
+  private volatile long cellsCountCompactedToMob = 0;
+  private volatile long cellsCountCompactedFromMob = 0;
+  private volatile long cellsSizeCompactedToMob = 0;
+  private volatile long cellsSizeCompactedFromMob = 0;
   private volatile long mobFlushCount = 0;
   private volatile long mobFlushedCellsCount = 0;
   private volatile long mobFlushedCellsSize = 0;
@@ -490,36 +490,36 @@ public class HMobStore extends HStore {
     }
   }
 
-  public void updateMobCompactedIntoMobCellsCount(long count) {
-    mobCompactedIntoMobCellsCount += count;
+  public void updateCellsCountCompactedToMob(long count) {
+    cellsCountCompactedToMob += count;
   }
 
-  public long getMobCompactedIntoMobCellsCount() {
-    return mobCompactedIntoMobCellsCount;
+  public long getCellsCountCompactedToMob() {
+    return cellsCountCompactedToMob;
   }
 
-  public void updateMobCompactedFromMobCellsCount(long count) {
-    mobCompactedFromMobCellsCount += count;
+  public void updateCellsCountCompactedFromMob(long count) {
+    cellsCountCompactedFromMob += count;
   }
 
-  public long getMobCompactedFromMobCellsCount() {
-    return mobCompactedFromMobCellsCount;
+  public long getCellsCountCompactedFromMob() {
+    return cellsCountCompactedFromMob;
   }
 
-  public void updateMobCompactedIntoMobCellsSize(long size) {
-    mobCompactedIntoMobCellsSize += size;
+  public void updateCellsSizeCompactedToMob(long size) {
+    cellsSizeCompactedToMob += size;
   }
 
-  public long getMobCompactedIntoMobCellsSize() {
-    return mobCompactedIntoMobCellsSize;
+  public long getCellsSizeCompactedToMob() {
+    return cellsSizeCompactedToMob;
   }
 
-  public void updateMobCompactedFromMobCellsSize(long size) {
-    mobCompactedFromMobCellsSize += size;
+  public void updateCellsSizeCompactedFromMob(long size) {
+    cellsSizeCompactedFromMob += size;
   }
 
-  public long getMobCompactedFromMobCellsSize() {
-    return mobCompactedFromMobCellsSize;
+  public long getCellsSizeCompactedFromMob() {
+    return cellsSizeCompactedFromMob;
   }
 
   public void updateMobFlushCount() {

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 24790e3..cf0d3f5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -82,10 +82,10 @@ class MetricsRegionServerWrapperImpl
   private volatile long flushedCellsSize = 0;
   private volatile long compactedCellsSize = 0;
   private volatile long majorCompactedCellsSize = 0;
-  private volatile long mobCompactedIntoMobCellsCount = 0;
-  private volatile long mobCompactedFromMobCellsCount = 0;
-  private volatile long mobCompactedIntoMobCellsSize = 0;
-  private volatile long mobCompactedFromMobCellsSize = 0;
+  private volatile long cellsCountCompactedToMob = 0;
+  private volatile long cellsCountCompactedFromMob = 0;
+  private volatile long cellsSizeCompactedToMob = 0;
+  private volatile long cellsSizeCompactedFromMob = 0;
   private volatile long mobFlushCount = 0;
   private volatile long mobFlushedCellsCount = 0;
   private volatile long mobFlushedCellsSize = 0;
@@ -449,23 +449,23 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
-  public long getMobCompactedFromMobCellsCount() {
-    return mobCompactedFromMobCellsCount;
+  public long getCellsCountCompactedFromMob() {
+    return cellsCountCompactedFromMob;
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsCount() {
-    return mobCompactedIntoMobCellsCount;
+  public long getCellsCountCompactedToMob() {
+    return cellsCountCompactedToMob;
   }
 
   @Override
-  public long getMobCompactedFromMobCellsSize() {
-    return mobCompactedFromMobCellsSize;
+  public long getCellsSizeCompactedFromMob() {
+    return cellsSizeCompactedFromMob;
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsSize() {
-    return mobCompactedIntoMobCellsSize;
+  public long getCellsSizeCompactedToMob() {
+    return cellsSizeCompactedToMob;
   }
 
   @Override
@@ -560,10 +560,10 @@ class MetricsRegionServerWrapperImpl
       long tempFlushedCellsSize = 0;
       long tempCompactedCellsSize = 0;
       long tempMajorCompactedCellsSize = 0;
-      long tempMobCompactedIntoMobCellsCount = 0;
-      long tempMobCompactedFromMobCellsCount = 0;
-      long tempMobCompactedIntoMobCellsSize = 0;
-      long tempMobCompactedFromMobCellsSize = 0;
+      long tempCellsCountCompactedToMob = 0;
+      long tempCellsCountCompactedFromMob = 0;
+      long tempCellsSizeCompactedToMob = 0;
+      long tempCellsSizeCompactedFromMob = 0;
       long tempMobFlushCount = 0;
       long tempMobFlushedCellsCount = 0;
       long tempMobFlushedCellsSize = 0;
@@ -596,10 +596,10 @@ class MetricsRegionServerWrapperImpl
           tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
           if (store instanceof HMobStore) {
             HMobStore mobStore = (HMobStore) store;
-            tempMobCompactedIntoMobCellsCount += mobStore.getMobCompactedIntoMobCellsCount();
-            tempMobCompactedFromMobCellsCount += mobStore.getMobCompactedFromMobCellsCount();
-            tempMobCompactedIntoMobCellsSize += mobStore.getMobCompactedIntoMobCellsSize();
-            tempMobCompactedFromMobCellsSize += mobStore.getMobCompactedFromMobCellsSize();
+            tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
+            tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
+            tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
+            tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
             tempMobFlushCount += mobStore.getMobFlushCount();
             tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
             tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
@@ -666,10 +666,10 @@ class MetricsRegionServerWrapperImpl
       flushedCellsSize = tempFlushedCellsSize;
       compactedCellsSize = tempCompactedCellsSize;
       majorCompactedCellsSize = tempMajorCompactedCellsSize;
-      mobCompactedIntoMobCellsCount = tempMobCompactedIntoMobCellsCount;
-      mobCompactedFromMobCellsCount = tempMobCompactedFromMobCellsCount;
-      mobCompactedIntoMobCellsSize = tempMobCompactedIntoMobCellsSize;
-      mobCompactedFromMobCellsSize = tempMobCompactedFromMobCellsSize;
+      cellsCountCompactedToMob = tempCellsCountCompactedToMob;
+      cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
+      cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
+      cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
       mobFlushCount = tempMobFlushCount;
       mobFlushedCellsCount = tempMobFlushedCellsCount;
       mobFlushedCellsSize = tempMobFlushedCellsSize;

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
index d1cca98..aa36e3e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ReversedMobStoreScanner.java
@@ -49,7 +49,7 @@ public class ReversedMobStoreScanner extends ReversedStoreScanner {
   }
 
   /**
-   * Firstly reads the cells from the HBase. If the cell are a reference cell (which has the
+   * Firstly reads the cells from the HBase. If the cell is a reference cell (which has the
    * reference tag), the scanner need seek this cell from the mob file, and use the cell found
    * from the mob file as the result.
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
index 8f94795..619a134 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
@@ -58,7 +58,7 @@ public class StoreFileInfo {
     Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
 
   /**
-   * A non-capture group, for hfiles, so that this can be embedded.
+   * A non-capture group, for del files, so that this can be embedded.
    * A del file has (_del) as suffix.
    */
   public static final String DELFILE_NAME_REGEX = "[0-9a-f]+(?:_del)";

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
index 175e8d8..e7c4dac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java
@@ -159,12 +159,12 @@ public class SnapshotManifest {
     RegionVisitor visitor = createRegionVisitor(desc);
 
     // 1. dump region meta info into the snapshot directory
-    LOG.debug("Storing '" + regionInfo + "' region-info for snapshot.");
+    LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot.");
     Object regionData = visitor.regionOpen(regionInfo);
     monitor.rethrowException();
 
     // 2. iterate through all the stores in the region
-    LOG.debug("Creating references for hfiles");
+    LOG.debug("Creating references for mob files");
 
     Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
     for (HColumnDescriptor hcd : hcds) {
@@ -188,7 +188,7 @@ public class SnapshotManifest {
         storeFiles.add(new StoreFileInfo(conf, fs, stat));
       }
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
+        LOG.debug("Adding snapshot references for " + storeFiles + " mob files");
       }
 
       // 2.2. iterate through all the mob files and create "references".
@@ -198,7 +198,7 @@ public class SnapshotManifest {
 
         // create "reference" to this store file.
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Adding reference for file (" + (i + 1) + "/" + sz + "): "
+          LOG.debug("Adding reference for mob file (" + (i + 1) + "/" + sz + "): "
             + storeFile.getPath());
         }
         visitor.storeFile(regionData, familyData, storeFile);