You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/08/04 21:14:24 UTC
[03/17] hive git commit: HIVE-17234 Remove HBase metastore from
master (Alan Gates, reviewed by Daniel Dai and Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/5e061557/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
deleted file mode 100644
index 06884b3..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/TestHBaseFilterPlanUtil.java
+++ /dev/null
@@ -1,483 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.hbase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.FilterPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.MultiScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
-import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan.ScanMarker;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
-import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.primitives.Shorts;
-
-public class TestHBaseFilterPlanUtil {
- final boolean INCLUSIVE = true;
-
- /**
- * Test the function that compares byte arrays
- */
- @Test
- public void testCompare() {
-
- Assert.assertEquals(-1, HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 3 }));
- Assert.assertEquals(-1,
- HBaseFilterPlanUtil.compare(new byte[] { 1, 2, 3 }, new byte[] { 1, 3 }));
- Assert.assertEquals(-1,
- HBaseFilterPlanUtil.compare(new byte[] { 1, 2 }, new byte[] { 1, 2, 3 }));
-
- Assert.assertEquals(0, HBaseFilterPlanUtil.compare(new byte[] { 3, 2 }, new byte[] { 3, 2 }));
-
- Assert
- .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 2, 1 }, new byte[] { 3, 2 }));
- Assert
- .assertEquals(1, HBaseFilterPlanUtil.compare(new byte[] { 3, 3, 1 }, new byte[] { 3, 2 }));
-
- }
-
- /**
- * Test function that finds greater/lesser marker
- */
- @Test
- public void testgetComparedMarker() {
- ScanMarker l;
- ScanMarker r;
-
- // equal plans
- l = new ScanMarker("1", INCLUSIVE, "int");
- r = new ScanMarker("1", INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- l = new ScanMarker("1", !INCLUSIVE, "int");
- r = new ScanMarker("1", !INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- assertFirstGreater(null, null);
-
- // create l is greater because of inclusive flag
- l = new ScanMarker("1", !INCLUSIVE, "int");
- // the rule for null vs non-null is different
- // non-null is both smaller and greater than null
- Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, true));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, true));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(l, null, false));
- Assert.assertEquals(l, ScanPlan.getComparedMarker(null, l, false));
-
- // create l that is greater because of the bytes
- l = new ScanMarker("2", INCLUSIVE, "int");
- r = new ScanMarker("1", INCLUSIVE, "int");
- assertFirstGreater(l, r);
-
- }
-
- private void assertFirstGreater(ScanMarker big, ScanMarker small) {
- Assert.assertEquals(big, ScanPlan.getComparedMarker(big, small, true));
- Assert.assertEquals(big, ScanPlan.getComparedMarker(small, big, true));
- Assert.assertEquals(small, ScanPlan.getComparedMarker(big, small, false));
- Assert.assertEquals(small, ScanPlan.getComparedMarker(small, big, false));
- }
-
- /**
- * Test ScanPlan AND operation
- */
- @Test
- public void testScanPlanAnd() {
- ScanPlan l = new ScanPlan();
- ScanPlan r = new ScanPlan();
- l.setStartMarker("a", "int", "10", INCLUSIVE);
- r.setStartMarker("a", "int", "10", INCLUSIVE);
-
- ScanPlan res;
- // both equal
- res = l.and(r).getPlans().get(0);
- Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
-
- // add equal end markers as well, and test AND again
- l.setEndMarker("a", "int", "20", INCLUSIVE);
- r.setEndMarker("a", "int", "20", INCLUSIVE);
- res = l.and(r).getPlans().get(0);
- Assert.assertEquals(new ScanMarker("10", INCLUSIVE, "int"), res.markers.get("a").startMarker);
- Assert.assertEquals(new ScanMarker("20", INCLUSIVE, "int"), res.markers.get("a").endMarker);
-
- l.setStartMarker("a", "int", "10", !INCLUSIVE);
- l.setEndMarker("a", "int", "20", INCLUSIVE);
-
- r.setStartMarker("a", "int", "10", INCLUSIVE);
- r.setEndMarker("a", "int", "15", INCLUSIVE);
- res = l.and(r).getPlans().get(0);
- // start of l is greater, end of r is smaller
- Assert.assertEquals(l.markers.get("a").startMarker, res.markers.get("a").startMarker);
- Assert.assertEquals(r.markers.get("a").endMarker, res.markers.get("a").endMarker);
-
- }
-
- /**
- * Test ScanPlan OR operation
- */
- @Test
- public void testScanPlanOr() {
- ScanPlan l = new ScanPlan();
- ScanPlan r = new ScanPlan();
- l.setStartMarker("a", "int", "1", INCLUSIVE);
- r.setStartMarker("a", "int", "11", INCLUSIVE);
-
- FilterPlan res1 = l.or(r);
- Assert.assertEquals(2, res1.getPlans().size());
- res1.getPlans().get(0).markers.get("a").startMarker.equals(l.markers.get("a").startMarker);
- res1.getPlans().get(1).markers.get("a").startMarker.equals(r.markers.get("a").startMarker);
-
- FilterPlan res2 = res1.or(r);
- Assert.assertEquals(3, res2.getPlans().size());
- }
-
- /**
- * Test MultiScanPlan OR
- */
- @Test
- public void testMultiScanPlanOr() {
-
- MultiScanPlan l = createMultiScanPlan(new ScanPlan());
- MultiScanPlan r = createMultiScanPlan(new ScanPlan());
- // verify OR of two multi plans with one plan each
- Assert.assertEquals(2, l.or(r).getPlans().size());
-
- // verify OR of multi plan with a single scanplan
- Assert.assertEquals(2, l.or(new ScanPlan()).getPlans().size());
- Assert.assertEquals(2, (new ScanPlan()).or(l).getPlans().size());
-
- // verify or of two multiplans with more than one scan plan
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- Assert.assertEquals(3, l.or(r).getPlans().size());
- Assert.assertEquals(3, r.or(l).getPlans().size());
-
- }
-
- private MultiScanPlan createMultiScanPlan(ScanPlan... scanPlans) {
- return new MultiScanPlan(Arrays.asList(scanPlans));
- }
-
- /**
- * Test MultiScanPlan AND
- */
- @Test
- public void testMultiScanPlanAnd() {
- MultiScanPlan l = createMultiScanPlan(new ScanPlan());
- MultiScanPlan r = createMultiScanPlan(new ScanPlan());
-
- // two MultiScanPlan with single scan plans should result in new FilterPlan
- // with just one scan
- Assert.assertEquals(1, l.and(r).getPlans().size());
-
- // l has one ScanPlan, r has two. AND result should have two
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- Assert.assertEquals(2, l.and(r).getPlans().size());
- Assert.assertEquals(2, r.and(l).getPlans().size());
-
- // l has 2 ScanPlans, r has 3. AND result should have 6
- l = createMultiScanPlan(new ScanPlan(), new ScanPlan());
- r = createMultiScanPlan(new ScanPlan(), new ScanPlan(), new ScanPlan());
- Assert.assertEquals(6, l.and(r).getPlans().size());
- Assert.assertEquals(6, r.and(l).getPlans().size());
- }
-
- /**
- * Test plan generation from LeafNode
- *
- * @throws MetaException
- */
- @Test
- public void testLeafNodePlan() throws MetaException {
-
- final String KEY = "k1";
- final String VAL = "v1";
- final String OTHERKEY = "k2";
- LeafNode l = new LeafNode();
- l.keyName = KEY;
- l.value = VAL;
- final ScanMarker DEFAULT_SCANMARKER = null;
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema(KEY, "int", null));
- parts.add(new FieldSchema(OTHERKEY, "int", null));
-
- l.operator = Operator.EQUALS;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), new ScanMarker(VAL, INCLUSIVE, "int"));
-
- l.operator = Operator.GREATERTHAN;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- l.operator = Operator.GREATERTHANOREQUALTO;
- verifyPlan(l, parts, KEY, new ScanMarker(VAL, INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- l.operator = Operator.LESSTHAN;
- verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, !INCLUSIVE, "int"));
-
- l.operator = Operator.LESSTHANOREQUALTO;
- verifyPlan(l, parts, KEY, DEFAULT_SCANMARKER, new ScanMarker(VAL, INCLUSIVE, "int"));
-
- // following leaf node plans should currently have true for 'has unsupported condition',
- // because of the condition is not on first key
- l.operator = Operator.EQUALS;
- verifyPlan(l, parts, OTHERKEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, false);
-
- // if tree is null, it should return equivalent of full scan, and true
- // for 'has unsupported condition'
- verifyPlan(null, parts, KEY, DEFAULT_SCANMARKER, DEFAULT_SCANMARKER, true);
-
- }
-
- private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker)
- throws MetaException {
- verifyPlan(l, parts, keyName, startMarker, endMarker, false);
- }
-
- private void verifyPlan(TreeNode l, List<FieldSchema> parts, String keyName, ScanMarker startMarker, ScanMarker endMarker,
- boolean hasUnsupportedCondition) throws MetaException {
- ExpressionTree e = null;
- if (l != null) {
- e = new ExpressionTree();
- e.setRootForTest(l);
- }
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- FilterPlan plan = planRes.plan;
- Assert.assertEquals("Has unsupported condition", hasUnsupportedCondition,
- planRes.hasUnsupportedCondition);
- Assert.assertEquals(1, plan.getPlans().size());
- ScanPlan splan = plan.getPlans().get(0);
- if (startMarker != null) {
- Assert.assertEquals(startMarker, splan.markers.get(keyName).startMarker);
- } else {
- Assert.assertTrue(splan.markers.get(keyName)==null ||
- splan.markers.get(keyName).startMarker==null);
- }
- if (endMarker != null) {
- Assert.assertEquals(endMarker, splan.markers.get(keyName).endMarker);
- } else {
- Assert.assertTrue(splan.markers.get(keyName)==null ||
- splan.markers.get(keyName).endMarker==null);
- }
- }
-
- /**
- * Test plan generation from TreeNode
- *
- * @throws MetaException
- */
- @Test
- public void testTreeNodePlan() throws MetaException {
-
- final String KEY = "k1";
- final String VAL1 = "10";
- final String VAL2 = "11";
- LeafNode l = new LeafNode();
- l.keyName = KEY;
- l.value = VAL1;
- final ScanMarker DEFAULT_SCANMARKER = null;
-
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("k1", "int", null));
-
- LeafNode r = new LeafNode();
- r.keyName = KEY;
- r.value = VAL2;
-
- TreeNode tn = new TreeNode(l, LogicalOperator.AND, r);
-
- // verify plan for - k1 >= '10' and k1 < '11'
- l.operator = Operator.GREATERTHANOREQUALTO;
- r.operator = Operator.LESSTHAN;
- verifyPlan(tn, parts, KEY, new ScanMarker(VAL1, INCLUSIVE, "int"), new ScanMarker(VAL2,
- !INCLUSIVE, "int"));
-
- // verify plan for - k1 >= '10' and k1 > '11'
- l.operator = Operator.GREATERTHANOREQUALTO;
- r.operator = Operator.GREATERTHAN;
- verifyPlan(tn, parts, KEY, new ScanMarker(VAL2, !INCLUSIVE, "int"), DEFAULT_SCANMARKER);
-
- // verify plan for - k1 >= '10' or k1 > '11'
- tn = new TreeNode(l, LogicalOperator.OR, r);
- ExpressionTree e = new ExpressionTree();
- e.setRootForTest(tn);
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(2, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
- // verify plan for - k1 >= '10' and (k1 >= '10' or k1 > '11')
- TreeNode tn2 = new TreeNode(l, LogicalOperator.AND, tn);
- e = new ExpressionTree();
- e.setRootForTest(tn2);
- planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(2, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
- // verify plan for (k1 >= '10' and (k1 >= '10' or k1 > '11')) or k1 LIKE '2'
- // plan should return true for hasUnsupportedCondition
- LeafNode klike = new LeafNode();
- klike.keyName = KEY;
- klike.value = VAL1;
- klike.operator = Operator.LIKE;
- TreeNode tn3 = new TreeNode(tn2, LogicalOperator.OR, klike);
- e = new ExpressionTree();
- e.setRootForTest(tn3);
- planRes = HBaseFilterPlanUtil.getFilterPlan(e, parts);
- Assert.assertEquals(3, planRes.plan.getPlans().size());
- Assert.assertEquals(false, planRes.hasUnsupportedCondition);
-
-
- }
-
- @Test
- public void testPartitionKeyScannerAllString() throws Exception {
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("year", "string", null));
- parts.add(new FieldSchema("month", "string", null));
- parts.add(new FieldSchema("state", "string", null));
-
- // One prefix key and one minor key range
- ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- ScanPlan sp = planRes.plan.getPlans().get(0);
- byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- RowFilter filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key year, rowfilter contains minor key state
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
- Assert.assertFalse(Bytes.contains(startRowSuffix, "CA".getBytes()));
- Assert.assertFalse(Bytes.contains(endRowSuffix, "CA".getBytes()));
-
- PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
-
- // Two prefix key and one LIKE operator
- exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and month > 10 "
- + "and month <= 11 and state like 'C%'").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key value year/month, rowfilter contains LIKE operator
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2015".getBytes()));
- Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "11".getBytes()));
-
- comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ops.size(), 1);
- Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
- // One prefix key, one minor key range and one LIKE operator
- exprTree = PartFilterExprUtil.getFilterParser("year >= 2014 and month > 10 "
- + "and month <= 11 and state like 'C%'").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key value year (low bound), rowfilter contains minor key state
- // and LIKE operator
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
-
- comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "month");
- Assert.assertEquals(comparator.ops.size(), 1);
- Assert.assertEquals(comparator.ops.get(0).keyName, "state");
-
- // Condition contains or
- exprTree = PartFilterExprUtil.getFilterParser("year = 2014 and (month > 10 "
- + "or month < 3)").tree;
- planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- sp = planRes.plan.getPlans().get(0);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // The first ScanPlan contains year = 2014 and month > 10
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(startRowSuffix, "10".getBytes()));
-
- sp = planRes.plan.getPlans().get(1);
- startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- filter = (RowFilter)sp.getFilter(parts);
-
- // The first ScanPlan contains year = 2014 and month < 3
- Assert.assertTrue(Bytes.contains(startRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "2014".getBytes()));
- Assert.assertTrue(Bytes.contains(endRowSuffix, "3".getBytes()));
- }
-
- @Test
- public void testPartitionKeyScannerMixedType() throws Exception {
- List<FieldSchema> parts = new ArrayList<FieldSchema>();
- parts.add(new FieldSchema("year", "int", null));
- parts.add(new FieldSchema("month", "int", null));
- parts.add(new FieldSchema("state", "string", null));
-
- // One prefix key and one minor key range
- ExpressionTree exprTree = PartFilterExprUtil.getFilterParser("year = 2015 and state = 'CA'").tree;
- PlanResult planRes = HBaseFilterPlanUtil.getFilterPlan(exprTree, parts);
-
- Assert.assertEquals(planRes.plan.getPlans().size(), 1);
-
- ScanPlan sp = planRes.plan.getPlans().get(0);
- byte[] startRowSuffix = sp.getStartRowSuffix("testdb", "testtb", parts);
- byte[] endRowSuffix = sp.getEndRowSuffix("testdb", "testtb", parts);
- RowFilter filter = (RowFilter)sp.getFilter(parts);
-
- // scan range contains the major key year, rowfilter contains minor key state
- Assert.assertTrue(Bytes.contains(startRowSuffix, Shorts.toByteArray((short)2015)));
- Assert.assertTrue(Bytes.contains(endRowSuffix, Shorts.toByteArray((short)2016)));
-
- PartitionKeyComparator comparator = (PartitionKeyComparator)filter.getComparator();
- Assert.assertEquals(comparator.ranges.size(), 1);
- Assert.assertEquals(comparator.ranges.get(0).keyName, "state");
- }
-}