You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2017/11/23 16:21:28 UTC
hbase git commit: HBASE-19266 TestAcidGuarantees should cover
adaptive in-memory compaction
Repository: hbase
Updated Branches:
refs/heads/branch-2 ca6e7e68f -> 66e650c39
HBASE-19266 TestAcidGuarantees should cover adaptive in-memory compaction
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66e650c3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66e650c3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66e650c3
Branch: refs/heads/branch-2
Commit: 66e650c39b7d794e9cd774add7b9f2be9d064e8f
Parents: ca6e7e6
Author: Chia-Ping Tsai <ch...@gmail.com>
Authored: Fri Nov 24 00:16:50 2017 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Fri Nov 24 00:19:06 2017 +0800
----------------------------------------------------------------------
.../hadoop/hbase/AcidGuaranteesTestTool.java | 3 +-
.../apache/hadoop/hbase/TestAcidGuarantees.java | 149 -------------------
.../TestAcidGuaranteesWithAdaptivePolicy.java | 35 +++++
.../TestAcidGuaranteesWithBasicPolicy.java | 31 ++++
.../TestAcidGuaranteesWithEagerPolicy.java | 35 +++++
...TestAcidGuaranteesWithNoInMemCompaction.java | 140 +++++++++++++++++
6 files changed, 242 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
index 5e00e8c..6a21438 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestTool.java
@@ -27,7 +27,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
-
import org.apache.commons.cli.CommandLine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -62,7 +61,7 @@ import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
@InterfaceAudience.Private
public class AcidGuaranteesTestTool extends AbstractHBaseTool {
- private static final Log LOG = LogFactory.getLog(TestAcidGuarantees.class);
+ private static final Log LOG = LogFactory.getLog(AcidGuaranteesTestTool.class);
public static final TableName TABLE_NAME = TableName.valueOf("TestAcidGuarantees");
public static final byte[] FAMILY_A = Bytes.toBytes("A");
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
deleted file mode 100644
index e42794e..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase;
-
-import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES;
-import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME;
-
-import java.util.List;
-import java.util.stream.Stream;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
-import org.apache.hadoop.hbase.testclassification.FlakeyTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-
-/**
- * Test case that uses multiple threads to read and write multifamily rows into a table, verifying
- * that reads never see partially-complete writes. This can run as a junit test, or with a main()
- * function which runs against a real cluster (eg for testing with failures, region movement, etc)
- */
-@Category({ FlakeyTests.class, LargeTests.class })
-@RunWith(Parameterized.class)
-public class TestAcidGuarantees {
-
- private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
-
- @Parameterized.Parameters(name = "{index}: compType={0}")
- public static Object[] data() {
- return new Object[] { "NONE", "BASIC", "EAGER" };
- }
-
- @Parameter
- public String compType;
-
- private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool();
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- // Set small flush size for minicluster so we exercise reseeking scanners
- Configuration conf = UTIL.getConfiguration();
- conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024));
- // prevent aggressive region split
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
- ConstantSizeRegionSplitPolicy.class.getName());
- conf.setInt("hfile.format.version", 3); // for mob tests
- UTIL.startMiniCluster(1);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- UTIL.shutdownMiniCluster();
- }
-
- @Before
- public void setUp() throws Exception {
- TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME)
- .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, compType);
- if (MemoryCompactionPolicy.valueOf(compType) == MemoryCompactionPolicy.EAGER) {
- builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false");
- builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9");
- }
- Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of)
- .forEachOrdered(builder::addColumnFamily);
- UTIL.getAdmin().createTable(builder.build());
- tool.setConf(UTIL.getConfiguration());
- }
-
- @After
- public void tearDown() throws Exception {
- UTIL.deleteTable(TABLE_NAME);
- }
-
- private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners,
- int numUniqueRows) throws Exception {
- runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false);
- }
-
- private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners,
- int numUniqueRows, boolean useMob) throws Exception {
- List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters",
- String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners",
- String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush");
- if (useMob) {
- args.add("-useMob");
- }
- tool.run(args.toArray(new String[0]));
- }
-
- @Test
- public void testGetAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 5, 0, 3);
- }
-
- @Test
- public void testScanAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 0, 5, 3);
- }
-
- @Test
- public void testMixedAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 2, 2, 3);
- }
-
- @Test
- public void testMobGetAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 5, 0, 3, true);
- }
-
- @Test
- public void testMobScanAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 0, 5, 3, true);
- }
-
- @Test
- public void testMobMixedAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 2, 2, 3, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
new file mode 100644
index 0000000..3bc417e
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
@@ -0,0 +1,35 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.testclassification.FlakeyTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.Ignore;
+import org.junit.experimental.categories.Category;
+
+@Category({ FlakeyTests.class, MediumTests.class })
+// TODO: HBASE-19266 disables this test as the adaptive policy causes the negative size of MemStore
+@Ignore
+public class TestAcidGuaranteesWithAdaptivePolicy extends TestAcidGuaranteesWithNoInMemCompaction {
+
+ @Override
+ protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
+ return MemoryCompactionPolicy.ADAPTIVE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
new file mode 100644
index 0000000..78737a8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
@@ -0,0 +1,31 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.experimental.categories.Category;
+
+@Category({ MediumTests.class })
+public class TestAcidGuaranteesWithBasicPolicy extends TestAcidGuaranteesWithNoInMemCompaction {
+
+ @Override
+ protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
+ return MemoryCompactionPolicy.BASIC;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
new file mode 100644
index 0000000..b2c87f6
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
@@ -0,0 +1,35 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.testclassification.FlakeyTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.Ignore;
+import org.junit.experimental.categories.Category;
+
+@Category({ FlakeyTests.class, MediumTests.class })
+// TODO: HBASE-19266 disables this test as the eager policy causes the negative size of MemStore
+@Ignore
+public class TestAcidGuaranteesWithEagerPolicy extends TestAcidGuaranteesWithNoInMemCompaction {
+
+ @Override
+ protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
+ return MemoryCompactionPolicy.EAGER;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/66e650c3/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
new file mode 100644
index 0000000..9ce2337
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
@@ -0,0 +1,140 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES;
+import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME;
+
+import java.util.List;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
+
+/**
+ * Test case that uses multiple threads to read and write multifamily rows into a table, verifying
+ * that reads never see partially-complete writes. This can run as a junit test, or with a main()
+ * function which runs against a real cluster (eg for testing with failures, region movement, etc)
+ */
+@Category({ MediumTests.class })
+public class TestAcidGuaranteesWithNoInMemCompaction {
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool();
+
+ protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
+ return MemoryCompactionPolicy.NONE;
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // Set small flush size for minicluster so we exercise reseeking scanners
+ Configuration conf = UTIL.getConfiguration();
+ conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024));
+ // prevent aggressive region split
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
+ ConstantSizeRegionSplitPolicy.class.getName());
+ conf.setInt("hfile.format.version", 3); // for mob tests
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ MemoryCompactionPolicy policy = getMemoryCompactionPolicy();
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+ .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, policy.name());
+ if (policy == MemoryCompactionPolicy.EAGER) {
+ builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false");
+ builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9");
+ }
+ Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of)
+ .forEachOrdered(builder::addColumnFamily);
+ UTIL.getAdmin().createTable(builder.build());
+ tool.setConf(UTIL.getConfiguration());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ UTIL.deleteTable(TABLE_NAME);
+ }
+
+ private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners,
+ int numUniqueRows) throws Exception {
+ runTestAtomicity(millisToRun, numWriters, numGetters, numScanners, numUniqueRows, false);
+ }
+
+ private void runTestAtomicity(long millisToRun, int numWriters, int numGetters, int numScanners,
+ int numUniqueRows, boolean useMob) throws Exception {
+ List<String> args = Lists.newArrayList("-millis", String.valueOf(millisToRun), "-numWriters",
+ String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters), "-numScanners",
+ String.valueOf(numScanners), "-numUniqueRows", String.valueOf(numUniqueRows), "-crazyFlush");
+ if (useMob) {
+ args.add("-useMob");
+ }
+ tool.run(args.toArray(new String[0]));
+ }
+
+ @Test
+ public void testGetAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 5, 0, 3);
+ }
+
+ @Test
+ public void testScanAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 0, 5, 3);
+ }
+
+ @Test
+ public void testMixedAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 2, 2, 3);
+ }
+
+ @Test
+ public void testMobGetAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 5, 0, 3, true);
+ }
+
+ @Test
+ public void testMobScanAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 0, 5, 3, true);
+ }
+
+ @Test
+ public void testMobMixedAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 2, 2, 3, true);
+ }
+}