You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/08 05:53:36 UTC
[11/19] kylin git commit: KYLIN-2255 drop v1 coprocessor impl
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
deleted file mode 100644
index 74b2112..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestFuzzyRowFilterV2 {
- @Test
- public void testSatisfiesNoUnsafeForward() {
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
- FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- 0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { 0, 0, 1, 0 })); // mask
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, 0, 5, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
- }
-
- @Test
- public void testSatisfiesForward() {
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
- FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { -1, -1, 0, -1 })); // mask
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 1, 2 }, new byte[] { 0, -1, -1 }));
- }
-
- @Test
- public void testSatisfiesReverse() {
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 2, 3, 1, 1, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) 245, 1, 3, 0 }, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 3, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 2, 1, 1, 1, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 1, 2 }, new byte[] { 0, -1, -1 }));
- }
-
- @Test
- public void testSatisfiesNoUnsafeReverse() {
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 2, 3, 1, 1, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) 245, 1, 3, 0 }, 0, 5, new byte[] { 1, 1, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 3, 1, 3, 0 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 2, 1, 1, 1, 0 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
- }
-
- @Test
- public void testGetNextForFuzzyRuleForward() {
- assertNext(false, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 2, 1, 0, 1 }, // current
- new byte[] { 2, 1, 2, 0, 0 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 1, 2, 0, 1 }, // current
- new byte[] { 1, 1, 2, 0, 2 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 2, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, -1, 0 }, // mask
- new byte[] { 1, 0, 2, 0, 1 }, // current
- new byte[] { 1, 1, 0, 2, 0 }); // expected next
-
- assertNext(false, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2, 0, 1 }, new byte[] { 1, (byte) 129, 1, 0, 0 });
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 1 }, new byte[] { 5, 1, 1, 1 });
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 1, 1 }, new byte[] { 5, 1, 0, 1, 2 });
-
- assertNext(false, new byte[] { 0, 1, 0, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, 0 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 5, 1, (byte) 255, 2 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 6, 1, 0, 1 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 0 }, // current
- new byte[] { 5, 1, (byte) 255, 1 }); // expected next
-
- assertNext(false, new byte[] { 5, 1, 1, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 5, 1, (byte) 255, 1 }, new byte[] { 5, 1, (byte) 255, 2 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 2, 2 }, new byte[] { 1, 1, 2, 3 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 3, 2 }, new byte[] { 1, 1, 3, 3 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 2, 3 }, new byte[] { 1, 1, 2, 4 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 3, 2 }, new byte[] { 1, 1, 3, 3 });
-
- assertNext(false, new byte[] { 1, 1, 0, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 0, 1, 3, 2 }, new byte[] { 1, 1, 0, 0 });
-
- // No next for this one
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 2, 3, 1, 1, 1 }, // row to check
- new byte[] { 1, 0, 1 }, // fuzzy row
- new byte[] { -1, 0, -1 })); // mask
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 1, (byte) 245, 1, 3, 0 }, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 1, 3, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 2, 1, 1, 1, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- }
-
- @Test
- public void testGetNextForFuzzyRuleReverse() {
- assertNext(true, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 2, 1, 0, 1 }, // current
- // TODO: should be {1, 1, 3} ?
- new byte[] { 1, 1, 2, (byte) 0xFF, (byte) 0xFF }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 2, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, -1, 0 }, // mask
- new byte[] { 1, 2, 1, 3, 1 }, // current
- // TODO: should be {1, 1, 1, 3} ?
- new byte[] { 1, 1, 0, 2, 0 }); // expected next
-
- assertNext(true, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2, 0, 1 },
- // TODO: should be {1, (byte) 128, 2} ?
- new byte[] { 1, (byte) 128, 1, (byte) 0xFF, (byte) 0xFF });
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2, 1 },
- // TODO: should be {5, 1, 0, 2} ?
- new byte[] { 5, 1, 0, 1, (byte) 0xFF });
-
- assertNext(true, new byte[] { 0, 1, 0, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, 0 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 5, 1, (byte) 255, 0 }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, 0, 1 }, // current
- new byte[] { 4, 1, (byte) 255, 1 }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 0 }, // current
- new byte[] { 5, 1, (byte) 254, 1 }); // expected next
-
- assertNext(true, new byte[] { 1, 1, 0, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 2, 1, 3, 2 },
- // TODO: should be {1, 0} ?
- new byte[] { 1, 1, 0, 0 });
-
- assertNext(true, new byte[] { 1, 0, 1 }, // fuzzy row
- new byte[] { -1, 0, -1 }, // mask
- new byte[] { 2, 3, 1, 1, 1 }, // row to check
- // TODO: should be {1, (byte) 0xFF, 2} ?
- new byte[] { 1, 0, 1, (byte) 0xFF, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, (byte) 245, 1, 3, 0 },
- // TODO: should be {1, 1, (byte) 255, 4} ?
- new byte[] { 1, 1, 0, 3, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, 3, 1, 3, 0 },
- // TODO: should be 1, 2, (byte) 255, 4 ?
- new byte[] { 1, 2, 0, 3, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 2, 1, 1, 1, 0 },
- // TODO: should be {1, 2, (byte) 255, 4} ?
- new byte[] { 1, 2, 0, 3, (byte) 0xFF });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2 }, new byte[] { 1, (byte) 128, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2 }, new byte[] { 5, 1, 0, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 5, 1, 1, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 5, 1, (byte) 0xFF, 1 }, new byte[] { 5, 1, (byte) 0xFF, 0 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 2, 2 }, new byte[] { 1, 1, 2, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 2, 3 }, new byte[] { 1, 1, 2, 2 });
-
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(true, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- }
-
- private static void assertNext(boolean reverse, byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) {
- KeyValue kv = KeyValue.createFirstOnRow(current);
- byte[] nextForFuzzyRule = FuzzyRowFilterV2.getNextForFuzzyRule(reverse, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), fuzzyRow, mask);
- Assert.assertEquals(Bytes.toStringBinary(expected), Bytes.toStringBinary(nextForFuzzyRule));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
deleted file mode 100644
index 1d85922..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-@Ignore
-public class TestFuzzyRowFilterV2EndToEnd {
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final static byte fuzzyValue = (byte) 63;
- private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterV2EndToEnd.class);
-
- private static int firstPartCardinality = 4;
- private static int secondPartCardinality = 4;
- private static int thirdPartCardinality = 4;
- private static int colQualifiersTotal = 1;
- private static int totalFuzzyKeys = thirdPartCardinality / 2;
-
- private static String table = "TestFuzzyRowFilterEndToEnd";
-
- /**
- * @throws java.lang.Exception
- */
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- conf.setInt("hbase.client.scanner.caching", 1000);
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
- // set no splits
- conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
-
- TEST_UTIL.startMiniCluster();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- // Nothing to do.
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After
- public void tearDown() throws Exception {
- // Nothing to do.
- }
-
- @Test
- public void testEndToEnd() throws Exception {
- String cf = "f";
-
- HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
-
- // 10 byte row key - (2 bytes 4 bytes 4 bytes)
- // 4 byte qualifier
- // 4 byte value
-
- for (int i0 = 0; i0 < firstPartCardinality; i0++) {
-
- for (int i1 = 0; i1 < secondPartCardinality; i1++) {
-
- for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
- byte[] rk = new byte[10];
-
- ByteBuffer buf = ByteBuffer.wrap(rk);
- buf.clear();
- buf.putShort((short) i0);
- buf.putInt(i1);
- buf.putInt(i2);
- for (int c = 0; c < colQualifiersTotal; c++) {
- byte[] cq = new byte[4];
- Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
-
- Put p = new Put(rk);
- p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
- ht.put(p);
- }
- }
- }
- }
-
- TEST_UTIL.flush();
-
- // test passes
- runTest1(ht);
- runTest2(ht);
-
- }
-
- private void runTest1(HTable hTable) throws IOException {
- // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
-
- byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
- for (int i = 0; i < totalFuzzyKeys; i++) {
- byte[] fuzzyKey = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
- buf.clear();
- buf.putShort((short) 2);
- for (int j = 0; j < 4; j++) {
- buf.put(fuzzyValue);
- }
- buf.putInt(i);
-
- Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
- list.add(pair);
- }
-
- int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
- FuzzyRowFilterV2 fuzzyRowFilter0 = new FuzzyRowFilterV2(list);
- // Filters are not stateless - we can't reuse them
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(list);
-
- // regular test
- runScanner(hTable, expectedSize, fuzzyRowFilter0);
- // optimized from block cache
- runScanner(hTable, expectedSize, fuzzyRowFilter1);
-
- }
-
- private void runTest2(HTable hTable) throws IOException {
- // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
-
- byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
-
- for (int i = 0; i < totalFuzzyKeys; i++) {
- byte[] fuzzyKey = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
- buf.clear();
- buf.putShort((short) (i * 2));
- for (int j = 0; j < 4; j++) {
- buf.put(fuzzyValue);
- }
- buf.putInt(i * 2);
-
- Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
- list.add(pair);
- }
-
- int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
-
- FuzzyRowFilterV2 fuzzyRowFilter0 = new FuzzyRowFilterV2(list);
- // Filters are not stateless - we can't reuse them
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(list);
-
- // regular test
- runScanner(hTable, expectedSize, fuzzyRowFilter0);
- // optimized from block cache
- runScanner(hTable, expectedSize, fuzzyRowFilter1);
-
- }
-
- private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
-
- String cf = "f";
- Scan scan = new Scan();
- scan.addFamily(cf.getBytes());
- scan.setFilter(filter);
- List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
- first.getScanner(scan);
- RegionScanner scanner = first.getScanner(scan);
- List<Cell> results = new ArrayList<Cell>();
- // Result result;
- long timeBeforeScan = System.currentTimeMillis();
- int found = 0;
- while (scanner.next(results)) {
- found += results.size();
- results.clear();
- }
- found += results.size();
- long scanTime = System.currentTimeMillis() - timeBeforeScan;
- scanner.close();
-
- LOG.info("\nscan time = " + scanTime + "ms");
- LOG.info("found " + found + " results\n");
-
- assertEquals(expectedSize, found);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void testFilterList() throws Exception {
- String cf = "f";
- String table = "TestFuzzyRowFiltersInFilterList";
- HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
-
- // 10 byte row key - (2 bytes 4 bytes 4 bytes)
- // 4 byte qualifier
- // 4 byte value
-
- for (int i1 = 0; i1 < 5; i1++) {
- for (int i2 = 0; i2 < 5; i2++) {
- byte[] rk = new byte[10];
-
- ByteBuffer buf = ByteBuffer.wrap(rk);
- buf.clear();
- buf.putShort((short) 2);
- buf.putInt(i1);
- buf.putInt(i2);
-
- // Each row contains 5 columns
- for (int c = 0; c < 5; c++) {
- byte[] cq = new byte[4];
- Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
-
- Put p = new Put(rk);
- p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
- ht.put(p);
- LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
- }
- }
- }
-
- TEST_UTIL.flush();
-
- // test passes if we get back 5 KV's (1 row)
- runTest(ht, 5);
-
- }
-
- @SuppressWarnings("unchecked")
- private void runTest(HTable hTable, int expectedSize) throws IOException {
- // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
- byte[] fuzzyKey1 = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
- buf.clear();
- buf.putShort((short) 2);
- for (int i = 0; i < 4; i++)
- buf.put(fuzzyValue);
- buf.putInt((short) 1);
- byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- byte[] fuzzyKey2 = new byte[10];
- buf = ByteBuffer.wrap(fuzzyKey2);
- buf.clear();
- buf.putShort((short) 2);
- buf.putInt((short) 2);
- for (int i = 0; i < 4; i++)
- buf.put(fuzzyValue);
-
- byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
-
- Pair<byte[], byte[]> pair1 = Pair.newPair(fuzzyKey1, mask1);
- Pair<byte[], byte[]> pair2 = Pair.newPair(fuzzyKey2, mask2);
-
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(Lists.newArrayList(pair1));
- FuzzyRowFilterV2 fuzzyRowFilter2 = new FuzzyRowFilterV2(Lists.newArrayList(pair2));
- // regular test - we expect 1 row back (5 KVs)
- runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
- }
-
- private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
- String cf = "f";
- Scan scan = new Scan();
- scan.addFamily(cf.getBytes());
- FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
- scan.setFilter(filterList);
-
- ResultScanner scanner = hTable.getScanner(scan);
- List<Cell> results = new ArrayList<Cell>();
- Result result;
- long timeBeforeScan = System.currentTimeMillis();
- while ((result = scanner.next()) != null) {
- for (Cell kv : result.listCells()) {
- LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
- results.add(kv);
- }
- }
- long scanTime = System.currentTimeMillis() - timeBeforeScan;
- scanner.close();
-
- LOG.info("scan time = " + scanTime + "ms");
- LOG.info("found " + results.size() + " results");
-
- assertEquals(expectedSize, results.size());
- }
-}