[GitHub] [hbase] VladRodionov commented on a change in pull request #921: HBASE-22749: Distributed MOB compactions

[GitBox <gi...@apache.org>]

VladRodionov commented on a change in pull request #921: HBASE-22749: Distributed MOB compactions
URL: https://github.com/apache/hbase/pull/921#discussion_r357837870
 
 

 ##########
 File path: hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestMobCompaction.java
 ##########
 @@ -0,0 +1,413 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.MobFileCleanerChore;
+import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
+import org.apache.hadoop.hbase.mob.FaultyMobStoreCompactor;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobStoreEngine;
+import org.apache.hadoop.hbase.mob.MobUtils;
+
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hbase.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * An integration test to detect regressions in HBASE-22749. Test creates
+ * MOB-enabled table, and runs in parallel, the following tasks: loads data,
+ * runs MOB compactions, runs MOB cleaning chore. The failure injections into MOB
+ * compaction cycle is implemented via specific sub-class of DefaultMobStoreCompactor -
+ * FaultyMobStoreCompactor. The probability of failure is controlled by command-line
+ * argument 'failprob'.
+ * @see <a href="https://issues.apache.org/jira/browse/HBASE-22749">HBASE-22749</a>
+ */
+@SuppressWarnings("deprecation")
+
+@Category(IntegrationTests.class)
+public class IntegrationTestMobCompaction extends IntegrationTestBase {
+  protected static final Logger LOG = LoggerFactory.getLogger(IntegrationTestMobCompaction.class);
+
+  protected static final String REGIONSERVER_COUNT_KEY = "servers";
+  protected static final String ROWS_COUNT_KEY = "rows";
+  protected static final String FAILURE_PROB_KEY = "failprob";
+
+  protected static final int DEFAULT_REGIONSERVER_COUNT = 3;
+  protected static final int DEFAULT_ROWS_COUNT = 5000000;
+  protected static final double DEFAULT_FAILURE_PROB = 0.1;
+
+  protected static int regionServerCount = DEFAULT_REGIONSERVER_COUNT;
+  protected static long rowsToLoad = DEFAULT_ROWS_COUNT;
+  protected static double failureProb = DEFAULT_FAILURE_PROB;
+
+  protected static String famStr = "f1";
+  protected static byte[] fam = Bytes.toBytes(famStr);
+  protected static byte[] qualifier = Bytes.toBytes("q1");
+  protected static long mobLen = 10;
+  protected static byte[] mobVal = Bytes
+      .toBytes("01234567890123456789012345678901234567890123456789012345678901234567890123456789");
+
+  private static Configuration conf;
+  private static HTableDescriptor hdt;
+  private static HColumnDescriptor hcd;
+  private static Admin admin;
+  private static Table table = null;
+  private static MobFileCleanerChore chore;
+
+  private static volatile boolean run = true;
+
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    util = getTestingUtil(getConf());
+    conf = util.getConfiguration();
+    // Initialize with test-specific configuration values
+    initConf(conf);
+    regionServerCount =
+        conf.getInt(REGIONSERVER_COUNT_KEY, DEFAULT_REGIONSERVER_COUNT);
+    LOG.info("Initializing cluster with {} region servers.", regionServerCount);
+    util.initializeCluster(regionServerCount);
+    admin = util.getAdmin();
+
+    createTestTable();
+
+    LOG.info("Cluster initialized and ready");
+  }
+
+  private void createTestTable() throws IOException {
+    // Create test table
+    hdt = util.createTableDescriptor("testMobCompactTable");
+    hcd = new HColumnDescriptor(fam);
+    hcd.setMobEnabled(true);
+    hcd.setMobThreshold(mobLen);
+    hcd.setMaxVersions(1);
+    hdt.addFamily(hcd);
+    table = util.createTable(hdt, null);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    LOG.info("Cleaning up after test.");
+    if(util.isDistributedCluster()) {
+      deleteTablesIfAny();
+      // TODO
+    }
+    LOG.info("Restoring cluster.");
+    util.restoreCluster();
+    LOG.info("Cluster restored.");
+  }
+
+  @Override
+  public void setUpMonkey() throws Exception {
 
 Review comment:
   Fixed.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services