You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/08/24 11:00:51 UTC
[12/22] incubator-kylin git commit: KYLIN-960 organize existing
storage module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
deleted file mode 100644
index bfeb783..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class ITHdfsOpsTest extends HBaseMetadataTestCase {
-
- FileSystem fileSystem;
-
- @Before
- public void setup() throws Exception {
-
- this.createTestMetadata();
-
- Configuration hconf = new Configuration();
-
- fileSystem = FileSystem.get(hconf);
- }
-
- @Test
- public void TestPath() throws IOException {
- String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
- Path coprocessorDir = new Path(hdfsWorkingDirectory, "test");
- fileSystem.mkdirs(coprocessorDir);
-
- Path newFile = new Path(coprocessorDir, "test_file");
- newFile = newFile.makeQualified(fileSystem.getUri(), null);
- FSDataOutputStream stream = fileSystem.create(newFile);
- stream.write(new byte[] { 0, 1, 2 });
- stream.close();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/RowValueDecoderTest.java
deleted file mode 100644
index 0fbdb34..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/RowValueDecoderTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.measure.LongMutable;
-import org.apache.kylin.metadata.measure.MeasureCodec;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class RowValueDecoderTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testDecode() throws Exception {
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
-
- MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
- BigDecimal sum = new BigDecimal("333.1234567");
- BigDecimal min = new BigDecimal("333.1111111");
- BigDecimal max = new BigDecimal("333.1999999");
- LongMutable count = new LongMutable(2);
- LongMutable item_count = new LongMutable(100);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- codec.encode(new Object[] { sum, min, max, count, item_count }, buf);
-
- buf.flip();
- byte[] valueBytes = new byte[buf.limit()];
- System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());
-
- RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- FunctionDesc aggrFunc = measure.getFunction();
- int index = hbaseCol.findMeasureIndex(aggrFunc);
- rowValueDecoder.setIndex(index);
- }
-
- rowValueDecoder.decode(valueBytes);
- Object[] measureValues = rowValueDecoder.getValues();
- //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
- assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testError() throws Exception {
- CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
- HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];
-
- MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
- BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
- BigDecimal min = new BigDecimal("333.1111111");
- BigDecimal max = new BigDecimal("333.1999999");
- LongWritable count = new LongWritable(2);
- LongMutable item_count = new LongMutable(100);
- ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
- codec.encode(new Object[] { sum, min, max, count, item_count }, buf);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
deleted file mode 100644
index fab9c7b..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceTool;
-import org.apache.kylin.common.util.ClassUtil;
-
-/**
- * This is a helper class for developer to directly manipulate the metadata store in sandbox
- * This is designed to run in IDE(i.e. not on sandbox hadoop CLI)
- *
- * For production metadata store manipulation refer to bin/metastore.sh in binary package
- * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
- */
-public class SandboxMetastoreCLI {
-
- private static final Log logger = LogFactory.getLog(SandboxMetastoreCLI.class);
-
- public static void main(String[] args) throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
-
- if (args.length < 2) {
- printUsage();
- return;
- }
-
- if ("download".equalsIgnoreCase(args[0])) {
- ResourceTool.main(new String[] { "download", args[1] });
- } else if ("upload".equalsIgnoreCase(args[0])) {
- ResourceTool.main(new String[] { "upload", args[1] });
- } else {
- printUsage();
- }
- }
-
- private static void printUsage() {
- logger.info("Usage: SandboxMetastoreCLI download toFolder");
- logger.info("Usage: SandboxMetastoreCLI upload fromFolder");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/TestHbaseClient.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/TestHbaseClient.java
deleted file mode 100644
index 0ea037b..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/TestHbaseClient.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.kylin.common.util.Bytes;
-
-/**
- */
-public class TestHbaseClient {
-
- private static boolean reverse = false;
-
- public static void foo(int n, int k) {
- int t = k;
- if (n - k < k) {
- t = n - k;
- reverse = true;
- }
- boolean[] flags = new boolean[n];
- inner(flags, 0, t);
- }
-
- private static void print(boolean[] flags) {
- for (int i = 0; i < flags.length; i++) {
- if (!reverse) {
- if (flags[i])
- System.out.print("0");
- else
- System.out.print("1");
- } else {
- if (flags[i])
- System.out.print("1");
- else
- System.out.print("0");
-
- }
- }
- System.out.println();
-
- }
-
- private static void inner(boolean[] flags, int start, int remaining) {
- if (remaining <= 0) {
- print(flags);
- return;
- }
-
- if (flags.length - start < remaining) {
- return;
- }
-
- // write at flags[start]
- flags[start] = true;
- inner(flags, start + 1, remaining - 1);
-
- // not write at flags[start]
- flags[start] = false;
- inner(flags, start + 1, remaining);
- }
-
- public static void main(String[] args) throws IOException {
- foo(6, 5);
- foo(5, 2);
- foo(3, 0);
-
- Configuration conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", "hbase_host");
- conf.set("zookeeper.znode.parent", "/hbase-unsecure");
-
- HTable table = new HTable(conf, "test1");
- Put put = new Put(Bytes.toBytes("row1"));
-
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
- put.add(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val2"));
-
- table.put(put);
- table.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
new file mode 100644
index 0000000..03377e8
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/FuzzyValueCombinationTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.translate.FuzzyValueCombination;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+/**
+ * @author yangli9
+ *
+ */
+public class FuzzyValueCombinationTest {
+
+ static final TableDesc table = new TableDesc();
+ static {
+ table.setName("table");
+ table.setDatabase("default");
+ }
+ static final TblColRef col1 = col(1, table);
+ static final TblColRef col2 = col(2, table);
+ static final TblColRef col3 = col(3, table);
+
+ @Test
+ public void testBasics() {
+ System.out.println("test basics ============================================================================");
+ Map<TblColRef, Set<String>> values = Maps.newHashMap();
+ values.put(col1, set("a", "b", "c"));
+ values.put(col2, set("x", "y", "z"));
+ List<Map<TblColRef, String>> result = FuzzyValueCombination.calculate(values, 10);
+ for (Map<TblColRef, String> item : result) {
+ System.out.println(item);
+ }
+ assertEquals(9, result.size());
+ }
+
+ @Test
+ public void testSomeNull() {
+ System.out.println("test some null ============================================================================");
+ Map<TblColRef, Set<String>> values = Maps.newHashMap();
+ values.put(col1, set("a", "b", "c"));
+ values.put(col2, set());
+ values.put(col3, set("x", "y", "z"));
+ List<Map<TblColRef, String>> result = FuzzyValueCombination.calculate(values, 10);
+ for (Map<TblColRef, String> item : result) {
+ System.out.println(item);
+ }
+ assertEquals(9, result.size());
+ }
+
+ @Test
+ public void testAllNulls() {
+ System.out.println("test all nulls ============================================================================");
+ Map<TblColRef, Set<String>> values = Maps.newHashMap();
+ values.put(col1, set());
+ values.put(col2, set());
+ values.put(col3, set());
+ List<Map<TblColRef, String>> result = FuzzyValueCombination.calculate(values, 10);
+ for (Map<TblColRef, String> item : result) {
+ System.out.println(item);
+ }
+ assertEquals(0, result.size());
+ }
+
+ @Test
+ public void testCap() {
+ System.out.println("test cap ============================================================================");
+ Map<TblColRef, Set<String>> values = Maps.newHashMap();
+ values.put(col1, set("1", "2", "3", "4"));
+ values.put(col2, set("a", "b", "c"));
+ values.put(col3, set("x", "y", "z"));
+ List<Map<TblColRef, String>> result = FuzzyValueCombination.calculate(values, 10);
+ for (Map<TblColRef, String> item : result) {
+ System.out.println(item);
+ }
+ assertEquals(9, result.size());
+ }
+
+ private static TblColRef col(int i, TableDesc t) {
+ ColumnDesc col = ColumnDesc.mockup(t, i, "Col" + i, "string");
+ return new TblColRef(col);
+ }
+
+ private Set<String> set(String... values) {
+ return new HashSet<String>(Arrays.asList(values));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveJDBCClientTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveJDBCClientTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveJDBCClientTest.java
new file mode 100644
index 0000000..1a20edc
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveJDBCClientTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+/**
+ * A simple test case to check whether the Hive JDBC interface can fulfill Kylin's need;
+ * Before run it, you need startup the hive server on localhost: $HIVE_HOME/bin/hiveserver2
+ * @author shaoshi
+ *
+ */
+public class HiveJDBCClientTest {
+
+ private static String driverName = "org.apache.hive.jdbc.HiveDriver";
+
+ private static String tempFileName = "/tmp/a.txt";
+
+ //@Before
+ public void setup() {
+
+ try {
+ Class.forName(driverName);
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ File testFile = new File(tempFileName);
+
+ if (!testFile.exists()) {
+
+ FileWriter writer;
+ try {
+ writer = new FileWriter(testFile);
+ writer.write("1 a\n");
+ writer.write("2 b\n");
+
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+ //@After
+ public void tearDown() {
+
+ }
+
+ protected Connection getHiveConnection() throws SQLException {
+
+ return DriverManager.getConnection("jdbc:hive2://localhost:10000/default", "hive", "");
+ }
+
+ //@Test
+ public void testConnectToHive() throws SQLException {
+
+ //replace "hive" here with the name of the user the queries should run as
+ Connection con = getHiveConnection();
+
+ Statement stmt = con.createStatement();
+ String tableName = "testHiveDriverTable";
+ stmt.execute("drop table if exists " + tableName);
+ stmt.execute("create table " + tableName + " (key int, value string) row format delimited fields terminated by ' '");
+ // show tables
+ String sql = "show tables '" + tableName + "'";
+ System.out.println("Running: " + sql);
+ ResultSet res = stmt.executeQuery(sql);
+ if (res.next()) {
+ System.out.println(res.getString(1));
+ }
+ res.close();
+
+ // describe table
+ sql = "describe " + tableName;
+ System.out.println("Running: " + sql);
+ res = stmt.executeQuery(sql);
+ while (res.next()) {
+ System.out.println(res.getString(1) + "\t" + res.getString(2));
+ }
+ res.close();
+
+ // load data into table
+ // NOTE: filepath has to be local to the hive server
+ // NOTE: /tmp/a.txt is a ctrl-A separated file with two fields per line
+ String filepath = tempFileName;
+ sql = "load data local inpath '" + filepath + "' into table " + tableName;
+ System.out.println("Running: " + sql);
+ stmt.execute(sql);
+
+ // select * query
+ sql = "select * from " + tableName;
+ System.out.println("Running: " + sql);
+ res = stmt.executeQuery(sql);
+ while (res.next()) {
+ System.out.println(String.valueOf(res.getInt(1)) + "\t" + res.getString(2));
+ }
+ res.close();
+
+ // regular hive query
+ sql = "select count(1) from " + tableName;
+ System.out.println("Running: " + sql);
+ res = stmt.executeQuery(sql);
+ while (res.next()) {
+ System.out.println(res.getInt(1));
+ }
+ res.close();
+ }
+
+ //@Test
+ public void testShowExtendedTable() throws SQLException {
+ Connection con = getHiveConnection();
+
+ String tableName = "testHiveDriverTable";
+ String sql = "show table extended like " + tableName + "";
+
+ ResultSet res = executeSQL(con, sql);
+ while (res.next()) {
+ System.out.println("--- " + res.getString("tab_name"));
+ }
+
+ sql = "describe extended " + tableName + "";
+ res = executeSQL(con, sql);
+ while (res.next()) {
+ System.out.println("---" + res.getString(1) + " | " + res.getString(2));
+ }
+
+ }
+
+ protected ResultSet executeSQL(Connection con, String sql) throws SQLException {
+
+ Statement stmt = con.createStatement();
+ System.out.println("Running: " + sql);
+ stmt.execute(sql);
+
+ return stmt.getResultSet();
+ }
+
+ public void runTests() throws SQLException {
+ try {
+ setup();
+ testConnectToHive();
+ testShowExtendedTable();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ tearDown();
+ }
+
+ }
+
+ public static void main(String[] args) throws SQLException {
+ HiveJDBCClientTest test = new HiveJDBCClientTest();
+ test.runTests();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
new file mode 100644
index 0000000..2edf11e
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/HiveMiniClusterTest.java
@@ -0,0 +1,131 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+//*/
+//
+//package org.apache.kylin.storage.minicluster;
+//
+//import java.io.File;
+//import java.io.IOException;
+//import java.sql.Connection;
+//import java.sql.DriverManager;
+//import java.sql.SQLException;
+//
+//import org.apache.commons.io.FileUtils;
+//import org.apache.hadoop.hdfs.MiniDFSCluster;
+//import org.apache.hadoop.hive.conf.HiveConf;
+//import org.apache.hadoop.hive.service.HiveInterface;
+//import org.apache.hadoop.mapred.JobConf;
+//import org.apache.hadoop.mapred.MiniMRCluster;
+//
+///**
+// * This is a test case to verify whether the query can be executed on Hive minicluster;
+// * You need set $HADOOP_HOME environment parameter before run it;
+// * @author shaoshi
+// *
+// */
+//public class HiveMiniClusterTest extends HiveJDBCClientTest {
+// public static final File HIVE_BASE_DIR = new File("target/hive");
+// public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir");
+// public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir");
+// public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb");
+// public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs");
+// public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
+// public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse");
+// public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata");
+// public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp");
+// protected HiveInterface client;
+//
+// protected MiniDFSCluster miniDFS;
+// protected MiniMRCluster miniMR;
+//
+// //@Before
+// public void setup() {
+// super.setup();
+// startHiveMiniCluster();
+// }
+//
+// protected void startHiveMiniCluster() {
+// //Create and configure location for hive to dump junk in target folder
+// try {
+// FileUtils.forceMkdir(HIVE_BASE_DIR);
+// FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
+// FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
+// FileUtils.forceMkdir(HIVE_LOGS_DIR);
+// FileUtils.forceMkdir(HIVE_TMP_DIR);
+// FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
+// FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
+// FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
+// } catch (IOException e1) {
+// e1.printStackTrace();
+// System.exit(1);
+// }
+//
+// System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true");
+// System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath());
+// System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath());
+// System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
+// System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath());
+// System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
+// System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath());
+// System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath());
+// System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
+//
+// // custom properties
+// System.setProperty("hive.server2.long.polling.timeout", "5000");
+//
+// HiveConf conf = new HiveConf();
+//
+// /* Build MiniDFSCluster */
+// try {
+// miniDFS = new MiniDFSCluster.Builder(conf).build();
+//
+// /* Build MiniMR Cluster */
+// int numTaskTrackers = 1;
+// int numTaskTrackerDirectories = 1;
+// String[] racks = null;
+// String[] hosts = null;
+// miniMR = new MiniMRCluster(numTaskTrackers, miniDFS.getFileSystem().getUri().toString(), numTaskTrackerDirectories, racks, hosts, new JobConf(conf));
+// JobConf jobConf = miniMR.createJobConf(new JobConf(conf));
+// System.out.println("-------" + jobConf.get("fs.defaultFS"));
+// System.out.println("-------" + miniDFS.getFileSystem().getUri().toString());
+// System.setProperty("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
+// } catch (IOException e) {
+// e.printStackTrace();
+// System.exit(1);
+// }
+// }
+//
+// //@After
+// public void tearDown() {
+// if (miniMR != null)
+// miniMR.shutdown();
+//
+// if (miniDFS != null)
+// miniDFS.shutdown();
+//
+// super.tearDown();
+// }
+//
+// protected Connection getHiveConnection() throws SQLException {
+// return DriverManager.getConnection("jdbc:hive2:///", "", "");
+// }
+//
+// public static void main(String[] args) throws SQLException {
+// HiveMiniClusterTest test = new HiveMiniClusterTest();
+// test.runTests();
+// }
+//}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java
new file mode 100644
index 0000000..1cc6f89
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/ITStorageTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageQueryFactory;
+import org.apache.kylin.storage.cache.StorageMockUtils;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class ITStorageTest extends HBaseMetadataTestCase {
+
+ private IStorageQuery storageEngine;
+ private CubeInstance cube;
+ private StorageContext context;
+
+ @BeforeClass
+ public static void setupResource() throws Exception {
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+
+ CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
+ cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
+ Assert.assertNotNull(cube);
+ storageEngine = StorageQueryFactory.createQuery(cube);
+ String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ context = new StorageContext();
+ context.setConnUrl(url);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test(expected = ScanOutOfLimitException.class)
+ @Ignore
+ public void testScanOutOfLimit() {
+ context.setThreshold(1);
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+
+ search(groups, aggregations, null, context);
+ }
+
+ @Test
+ public void test01() {
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ /*
+ @Test
+ public void test02() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildFilter2(groups.get(1));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test03() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildAndFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test04() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildOrFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test05() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+
+ int count = search(groups, aggregations, null, context);
+ assertTrue(count > 0);
+ }
+ */
+ private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
+ int count = 0;
+ ITupleIterator iterator = null;
+ try {
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
+ iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
+ while (iterator.hasNext()) {
+ ITuple tuple = iterator.next();
+ System.out.println("Tuple = " + tuple);
+ count++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (iterator != null)
+ iterator.close();
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
new file mode 100644
index 0000000..949e22a
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+
+public class MiniClusterTest {
+
+ private static HBaseTestingUtility testUtil = new HBaseTestingUtility();
+
+ public static void main(String[] args) throws Exception {
+
+ File miniclusterFolder = new File(AbstractKylinTestCase.MINICLUSTER_TEST_DATA);
+ System.out.println("----" + miniclusterFolder.getAbsolutePath());
+
+ //save the dfs data to minicluster folder
+ System.setProperty("test.build.data", miniclusterFolder.getAbsolutePath());
+
+ MiniHBaseCluster hbCluster = testUtil.startMiniCluster(1);
+ testUtil.startMiniMapReduceCluster();
+ System.out.println("Minicluster started.");
+
+ Configuration conf = hbCluster.getConf();
+ String host = conf.get(HConstants.ZOOKEEPER_QUORUM);
+ String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ String parent = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
+
+ // reduce rpc retry
+ conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
+ conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
+ conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
+
+ String connectionUrl = "hbase:" + host + ":" + port + ":" + parent;
+
+ System.out.println("hbase connection url:" + connectionUrl);
+
+ testUtil.getDFSCluster().getFileSystem();
+ testUtil.shutdownMiniMapReduceCluster();
+ testUtil.shutdownMiniCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
new file mode 100644
index 0000000..f85bb8e
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TimeConditionLiteralsReplacerTest.java
@@ -0,0 +1,38 @@
+package org.apache.kylin.storage.hbase.common;
+
+import org.apache.kylin.dict.DictCodeSystem;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.TimeConditionLiteralsReplacer;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.common.coprocessor.FilterBaseTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ */
+public class TimeConditionLiteralsReplacerTest extends FilterBaseTest {
+ @Test
+ public void basicTest() {
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef column = new TblColRef(c1);
+
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ constantFilter = new ConstantTupleFilter("946684800000");
+ compareFilter.addChild(constantFilter);
+
+ TimeConditionLiteralsReplacer filterDecorator = new TimeConditionLiteralsReplacer(compareFilter);
+ byte[] bytes = TupleFilterSerializer.serialize(compareFilter, filterDecorator, DictCodeSystem.INSTANCE);
+ CompareTupleFilter compareTupleFilter = (CompareTupleFilter) TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
+ Assert.assertEquals("2000-01-01", compareTupleFilter.getFirstValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
new file mode 100644
index 0000000..335e00c
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/TsConditionEraserTest.java
@@ -0,0 +1,138 @@
+package org.apache.kylin.storage.hbase.common;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.StringCodeSystem;
+import org.apache.kylin.metadata.filter.TsConditionEraser;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class TsConditionEraserTest extends LocalFileMetadataTestCase {
+ IIInstance ii;
+ TableRecordInfo tableRecordInfo;
+ CoprocessorFilter filter;
+ TableDesc factTableDesc;
+
+ TblColRef caldt;
+ TblColRef siteId;
+
+ @Before
+ public void setup() throws IOException {
+ this.createTestMetadata();
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+ this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
+ this.factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ this.caldt = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");
+ this.siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");
+ }
+
+ @After
+ public void cleanUp() {
+ cleanupTestMetadata();
+ }
+
+ private TupleFilter mockFilter1(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ private TupleFilter mockFilter2(int year) {
+ CompareTupleFilter aFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ aFilter.addChild(new ColumnTupleFilter(caldt));
+ aFilter.addChild(new ConstantTupleFilter(year + "-01-01"));
+
+ CompareTupleFilter bFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ bFilter.addChild(new ColumnTupleFilter(caldt));
+ bFilter.addChild(new ConstantTupleFilter(year + "-01-04"));
+
+ CompareTupleFilter cFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ cFilter.addChild(new ColumnTupleFilter(caldt));
+ cFilter.addChild(new ConstantTupleFilter(year + "-01-03"));
+
+ CompareTupleFilter dFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ dFilter.addChild(new ColumnTupleFilter(siteId));
+ dFilter.addChild(new ConstantTupleFilter("0"));
+
+ LogicalTupleFilter subRoot = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ subRoot.addChildren(Lists.newArrayList(aFilter, bFilter, cFilter, dFilter));
+
+ CompareTupleFilter outFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
+ outFilter.addChild(new ColumnTupleFilter(caldt));
+ outFilter.addChild(new ConstantTupleFilter(year + "-01-02"));
+
+ LogicalTupleFilter root = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ root.addChildren(Lists.newArrayList(subRoot, outFilter));
+ return root;
+ }
+
+ @Test
+ public void positiveTest() {
+
+ TupleFilter a = mockFilter1(2000);
+ TupleFilter b = mockFilter1(2001);
+
+ TsConditionEraser decoratorA = new TsConditionEraser(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ TsConditionEraser decoratorB = new TsConditionEraser(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertArrayEquals(aBytes, bBytes);
+
+ }
+
+ @Test
+ public void negativeTest() {
+ TupleFilter a = mockFilter2(2000);
+ TupleFilter b = mockFilter2(2001);
+
+ TsConditionEraser decoratorA = new TsConditionEraser(caldt, a);
+ byte[] aBytes = TupleFilterSerializer.serialize(a, decoratorA, StringCodeSystem.INSTANCE);
+ TsConditionEraser decoratorB = new TsConditionEraser(caldt, b);
+ byte[] bBytes = TupleFilterSerializer.serialize(b, decoratorB, StringCodeSystem.INSTANCE);
+ Assert.assertFalse(Arrays.equals(aBytes, bBytes));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
new file mode 100644
index 0000000..483b8be
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterBaseTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.kylin.metadata.filter.CaseTupleFilter;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.DynamicTupleFilter;
+import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.StringCodeSystem;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+
+/**
+ * @author xjiang
+ *
+ */
+public class FilterBaseTest {
+
+ @SuppressWarnings("rawtypes")
+ static final IFilterCodeSystem CS = StringCodeSystem.INSTANCE;
+
+ protected List<TblColRef> buildGroups() {
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "string");
+ TblColRef cf1 = new TblColRef(c1);
+ groups.add(cf1);
+
+ TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
+ ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
+ TblColRef cf2 = new TblColRef(c2);
+ groups.add(cf2);
+
+ return groups;
+ }
+
+ protected CompareTupleFilter buildCompareFilter(List<TblColRef> groups, int index) {
+ TblColRef column = groups.get(index);
+ CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter);
+ ConstantTupleFilter constantFilter = null;
+ if (index == 0) {
+ constantFilter = new ConstantTupleFilter("2013-03-10");
+ } else if (index == 1) {
+ constantFilter = new ConstantTupleFilter("ClothinShoes & Accessories");
+ }
+ compareFilter.addChild(constantFilter);
+ return compareFilter;
+ }
+
+ protected TupleFilter buildAndFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
+ CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter.addChild(compareFilter1);
+ andFilter.addChild(compareFilter2);
+ return andFilter;
+ }
+
+ protected TupleFilter buildOrFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
+ CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
+ LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ logicFilter.addChild(compareFilter1);
+ logicFilter.addChild(compareFilter2);
+ return logicFilter;
+ }
+
+ protected CaseTupleFilter buildCaseFilter(List<TblColRef> groups) {
+ CaseTupleFilter caseFilter = new CaseTupleFilter();
+
+ TupleFilter when0 = buildAndFilter(groups);
+ caseFilter.addChild(when0);
+ TupleFilter then0 = new ConstantTupleFilter("0");
+ caseFilter.addChild(then0);
+
+ TupleFilter when1 = buildCompareFilter(groups, 0);
+ caseFilter.addChild(when1);
+ TupleFilter then1 = new ConstantTupleFilter("1");
+ caseFilter.addChild(then1);
+
+ TupleFilter when2 = buildCompareFilter(groups, 1);
+ caseFilter.addChild(when2);
+ TupleFilter then2 = new ConstantTupleFilter("2");
+ caseFilter.addChild(then2);
+
+ TupleFilter when3 = buildOrFilter(groups);
+ caseFilter.addChild(when3);
+ TupleFilter then3 = new ConstantTupleFilter("3");
+ caseFilter.addChild(then3);
+
+ TupleFilter else4 = new ConstantTupleFilter("4");
+ caseFilter.addChild(else4);
+
+ return caseFilter;
+ }
+
+ protected CompareTupleFilter buildCompareCaseFilter(List<TblColRef> groups, String constValue) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
+ CaseTupleFilter caseFilter = buildCaseFilter(groups);
+ compareFilter.addChild(caseFilter);
+ ConstantTupleFilter constantFilter = new ConstantTupleFilter(constValue);
+ compareFilter.addChild(constantFilter);
+ return compareFilter;
+ }
+
+ protected CompareTupleFilter buildCompareDynamicFilter(List<TblColRef> groups) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
+ compareFilter.addChild(new ColumnTupleFilter(groups.get(0)));
+ compareFilter.addChild(new DynamicTupleFilter("?0"));
+ compareFilter.bindVariable("?0", "abc");
+ return compareFilter;
+ }
+
+ protected void compareFilter(TupleFilter f1, TupleFilter f2) {
+ if (f1 == null && f2 == null) {
+ return;
+ }
+
+ if (f1 == null || f2 == null) {
+ throw new IllegalStateException("f1=" + f1 + ", f2=" + f2);
+ }
+
+ String str1 = f1.toString();
+ System.out.println("f1=" + str1);
+ String str2 = f2.toString();
+ System.out.println("f2=" + str2);
+ if (!str1.equals(str2)) {
+ throw new IllegalStateException("f1=" + str1 + ", f2=" + str2);
+ }
+
+ int s1 = f1.getChildren().size();
+ int s2 = f2.getChildren().size();
+ if (s1 != s2) {
+ throw new IllegalStateException("f1=" + str1 + ", f2=" + str2 + " has different children: " + s1 + " vs. " + s2);
+ }
+
+ for (int i = 0; i < s1; i++) {
+ compareFilter(f1.getChildren().get(i), f2.getChildren().get(i));
+ }
+ }
+
+ private static String[][] SAMPLE_DATA = new String[][] { { "2013-03-10", "2012-01-12", "2014-03-10" }, { "ClothinShoes & Accessories", "ABIN", "FP-GTC", "FP-NON-GTC" } };
+
+ protected Collection<Tuple> generateTuple(int number, List<TblColRef> columns, int[] matches) {
+
+ Collection<Tuple> tuples = new ArrayList<Tuple>(number);
+ TupleInfo info = new TupleInfo();
+ for (int i = 0; i < columns.size(); i++) {
+ TblColRef column = columns.get(i);
+ info.setField(column.getName(), column, i);
+ }
+
+ int allMatches = 0;
+ Random rand = new Random();
+ for (int i = 0; i < number; i++) {
+ Tuple t = new Tuple(info);
+ boolean isFullMatch = true;
+ for (int k = 0; k < columns.size(); k++) {
+ TblColRef column = columns.get(k);
+ int index = Math.abs(rand.nextInt()) % SAMPLE_DATA[k].length;
+ t.setDimensionValue(column.getName(), SAMPLE_DATA[k][index]);
+ if (index == 0) {
+ matches[k]++;
+ } else {
+ isFullMatch = false;
+ }
+ }
+ if (isFullMatch) {
+ allMatches++;
+ }
+ tuples.add(t);
+ }
+ matches[2] = allMatches;
+ return tuples;
+ }
+
+ protected int evaluateTuples(Collection<Tuple> tuples, TupleFilter filter) {
+ int match = 0;
+ for (Tuple t : tuples) {
+ if (filter.evaluate(t, CS)) {
+ match++;
+ }
+ }
+ return match;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
new file mode 100644
index 0000000..c395fd9
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterEvaluateTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.junit.Test;
+
+/**
+ * @author xjiang
+ *
+ */
+public class FilterEvaluateTest extends FilterBaseTest {
+
+ @Test
+ public void testEvaluate00() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[0]);
+ }
+
+ @Test
+ public void testEvaluate01() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareFilter(groups, 1);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[1]);
+ }
+
+ @Test
+ public void testEvaluate02() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildOrFilter(groups);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[0] + matcheCounts[1] - matcheCounts[2]);
+ }
+
+ @Test
+ public void testEvaluate03() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildAndFilter(groups);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[2]);
+ }
+
+ @Test
+ public void testEvaluate04() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "0");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[2]);
+ }
+
+ @Test
+ public void testEvaluate05() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "1");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[0] - matcheCounts[2]);
+ }
+
+ @Test
+ public void testEvaluate06() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "2");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 1;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, matcheCounts[1] - matcheCounts[2]);
+ }
+
+ @Test
+ public void testEvaluate07() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "3");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, 0);
+ }
+
+ @Test
+ public void testEvaluate08() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "4");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ int number = 10000;
+ int[] matcheCounts = new int[] { 0, 0, 0 };
+ Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
+ int match = evaluateTuples(tuples, newFilter);
+
+ assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
new file mode 100644
index 0000000..8bf8ecb
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterSerializeTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.Test;
+
+/**
+ * @author xjiang
+ *
+ */
+public class FilterSerializeTest extends FilterBaseTest {
+
+ @Test
+ public void testSerialize01() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize02() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareFilter(groups, 1);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize03() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildAndFilter(groups);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize04() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildOrFilter(groups);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize05() {
+ ColumnDesc column = new ColumnDesc();
+
+ TblColRef colRef = new TblColRef(column);
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+ groups.add(colRef);
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize06() {
+ ColumnDesc column = new ColumnDesc();
+ column.setName("META_CATEG_NAME");
+ TblColRef colRef = new TblColRef(column);
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+ groups.add(colRef);
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize07() {
+ TableDesc table = new TableDesc();
+ table.setName("TEST_KYLIN_FACT");
+ table.setDatabase("DEFAULT");
+
+ ColumnDesc column = new ColumnDesc();
+ column.setTable(table);
+ TblColRef colRef = new TblColRef(column);
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+ groups.add(colRef);
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize08() {
+ TableDesc table = new TableDesc();
+ table.setDatabase("DEFAULT");
+
+ ColumnDesc column = new ColumnDesc();
+ column.setTable(table);
+ TblColRef colRef = new TblColRef(column);
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+ groups.add(colRef);
+ TupleFilter filter = buildCompareFilter(groups, 0);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize10() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter orFilter = buildOrFilter(groups);
+ TupleFilter andFilter = buildAndFilter(groups);
+
+ LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ logicFilter.addChild(orFilter);
+ logicFilter.addChild(andFilter);
+
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(logicFilter, newFilter);
+ }
+
+ @Test
+ public void testSerialize11() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter orFilter = buildOrFilter(groups);
+ TupleFilter andFilter = buildAndFilter(groups);
+
+ LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ logicFilter.addChild(orFilter);
+ logicFilter.addChild(andFilter);
+
+ byte[] bytes = TupleFilterSerializer.serialize(logicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(logicFilter, newFilter);
+ }
+
+ @Test
+ public void testSerialize12() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCaseFilter(groups);
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testSerialize13() {
+ List<TblColRef> groups = buildGroups();
+ TupleFilter filter = buildCompareCaseFilter(groups, "0");
+
+ byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(filter, newFilter);
+ }
+
+ @Test
+ public void testDynamic() {
+ final CompareTupleFilter compareDynamicFilter = buildCompareDynamicFilter(buildGroups());
+
+ byte[] bytes = TupleFilterSerializer.serialize(compareDynamicFilter, CS);
+ TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);
+
+ bytes = TupleFilterSerializer.serialize(newFilter, CS);
+ TupleFilter newFilter2 = TupleFilterSerializer.deserialize(bytes, CS);
+
+ compareFilter(compareDynamicFilter, newFilter);
+ compareFilter(compareDynamicFilter, newFilter2);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3d2685d8/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/RowProjectorTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/RowProjectorTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/RowProjectorTest.java
new file mode 100644
index 0000000..6be7808
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/coprocessor/RowProjectorTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.common.coprocessor;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.kylin.common.util.Bytes;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * @author yangli9
+ *
+ */
+public class RowProjectorTest {
+
+ byte[] mask = new byte[] { (byte) 0xff, 0x00, 0x00, (byte) 0xff };
+ CoprocessorProjector sample = new CoprocessorProjector(mask, true);
+
+ @Test
+ public void testSerialize() {
+
+ byte[] bytes = CoprocessorProjector.serialize(sample);
+ CoprocessorProjector copy = CoprocessorProjector.deserialize(bytes);
+
+ assertTrue(Arrays.equals(sample.groupByMask, copy.groupByMask));
+ }
+
+ @Test
+ public void testProject() {
+ byte[] bytes1 = new byte[] { -1, -2, -3, -4 };
+ byte[] bytes2 = new byte[] { 1, 2, 3, 4 };
+ byte[] bytes3 = new byte[] { 1, 99, 100, 4 };
+ byte[] bytes4 = new byte[] { 1, 1, 1, 5 };
+
+ AggrKey rowKey = sample.getAggrKey(newCellWithRowKey(bytes1));
+ AggrKey rowKey2 = sample.getAggrKey(newCellWithRowKey(bytes2));
+ assertTrue(rowKey == rowKey2); // no extra object creation
+ assertTrue(Bytes.equals(rowKey.get(), rowKey.offset(), rowKey.length(), bytes2, 0, bytes2.length));
+
+ rowKey2 = rowKey.copy(); // explicit object creation
+ assertTrue(rowKey != rowKey2);
+
+ rowKey = sample.getAggrKey(newCellWithRowKey(bytes1));
+ assertTrue(rowKey.hashCode() != rowKey2.hashCode());
+ assertTrue(rowKey.equals(rowKey2) == false);
+ assertTrue(rowKey.compareTo(rowKey2) > 0); // unsigned compare
+
+ rowKey = sample.getAggrKey(newCellWithRowKey(bytes3));
+ assertTrue(rowKey.hashCode() == rowKey2.hashCode());
+ assertTrue(rowKey.equals(rowKey2) == true);
+ assertTrue(rowKey.compareTo(rowKey2) == 0);
+
+ rowKey = sample.getAggrKey(newCellWithRowKey(bytes4));
+ assertTrue(rowKey.hashCode() != rowKey2.hashCode());
+ assertTrue(rowKey.equals(rowKey2) == false);
+ assertTrue(rowKey.compareTo(rowKey2) > 0);
+ }
+
+ private List<Cell> newCellWithRowKey(byte[] rowkey) {
+ ArrayList<Cell> list = Lists.newArrayList();
+ list.add(new KeyValue(rowkey, null, null, null));
+ return list;
+ }
+}