You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/25 08:11:37 UTC
[05/11] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
KYLIN-697 non-integration tests all passed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/42f124fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/42f124fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/42f124fb
Branch: refs/heads/new697
Commit: 42f124fbbc249683f65d152078ae24ecd364bfa4
Parents: 08eedde
Author: honma <ho...@ebay.com>
Authored: Mon May 25 11:13:46 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:24:16 2015 +0800
----------------------------------------------------------------------
.../persistence/HBaseResourceStoreTest.java | 208 --------------
.../persistence/ITHBaseResourceStoreTest.java | 208 ++++++++++++++
.../kylin/common/util/ITMockHTableTest.java | 158 +++++++++++
.../kylin/common/util/MockHTableTest.java | 158 -----------
.../apache/kylin/dict/HiveTableReaderTest.java | 51 ----
.../kylin/dict/ITHiveTableReaderTest.java | 51 ++++
.../kylin/dict/ITSnapshotManagerTest.java | 82 ++++++
.../apache/kylin/dict/SnapshotManagerTest.java | 82 ------
.../Installation/Off Hadoop CLI Installation.md | 2 +-
.../MISC/FAQ on Kylin Installation and Usage.md | 2 +-
...n Kylin test case with HBase Mini Cluster.md | 2 +-
jdbc/kylin_jdbc.log.2014-12-22 | 18 ++
.../kylin/job/BuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/BuildCubeWithStreamTest.java | 224 ---------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 -----------------
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ------------------
.../apache/kylin/job/IIStreamBuilderTest.java | 108 -------
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 +++++++++++++++
.../kylin/job/ITBuildIIWithEngineTest.java | 254 +++++++++++++++++
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../apache/kylin/job/ITIIStreamBuilderTest.java | 108 +++++++
.../impl/threadpool/DefaultSchedulerTest.java | 102 -------
.../impl/threadpool/ITDefaultSchedulerTest.java | 97 +++++++
.../tool/HiveSourceTableLoaderTest.java | 59 ----
.../tool/ITHiveSourceTableLoaderTest.java | 59 ++++
pom.xml | 23 +-
.../kylin/query/test/CombinationTest.java | 72 -----
.../apache/kylin/query/test/IIQueryTest.java | 85 ------
.../kylin/query/test/ITCombinationTest.java | 72 +++++
.../apache/kylin/query/test/ITIIQueryTest.java | 85 ++++++
.../kylin/query/test/ITKylinQueryTest.java | 234 ++++++++++++++++
.../apache/kylin/query/test/KylinQueryTest.java | 234 ----------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 245 ++++++++++++++++
.../org/apache/kylin/jdbc/JDBCDriverTest.java | 245 ----------------
.../rest/controller/ITTableControllerTest.java | 74 +++++
.../rest/controller/TableControllerTest.java | 74 -----
.../kylin/rest/service/CacheServiceTest.java | 11 +-
.../storage/hbase/ITInvertedIndexHBaseTest.java | 114 ++++++++
.../storage/hbase/InvertedIndexHBaseTest.java | 114 --------
.../kylin/storage/test/ITStorageTest.java | 154 ++++++++++
.../apache/kylin/storage/test/StorageTest.java | 154 ----------
.../kylin/streaming/ITKafkaConsumerTest.java | 108 +++++++
.../kylin/streaming/ITKafkaRequesterTest.java | 76 +++++
.../kylin/streaming/KafkaConsumerTest.java | 108 -------
.../kylin/streaming/KafkaRequesterTest.java | 76 -----
46 files changed, 2979 insertions(+), 2988 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
deleted file mode 100644
index f4788f4..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.*;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
new file mode 100644
index 0000000..6195423
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+
+public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHBaseStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ @Test
+ public void testHBaseStoreWithLargeCell() throws Exception {
+ String path = "/cube/_test_large_cell.json";
+ String largeContent = "THIS_IS_A_LARGE_CELL";
+ StringEntity content = new StringEntity(largeContent);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int origSize = config.getHBaseKeyValueSize();
+ ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+
+ try {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+
+ store.deleteResource(path);
+
+ store.putResource(path, content, StringEntity.serializer);
+ assertTrue(store.exists(path));
+ StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
+ assertEquals(content, t);
+
+ Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+ assertTrue(fileSystem.exists(redirectPath));
+
+ FSDataInputStream in = fileSystem.open(redirectPath);
+ assertEquals(largeContent, in.readUTF());
+ in.close();
+
+ store.deleteResource(path);
+ } finally {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ store.deleteResource(path);
+ }
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
new file mode 100644
index 0000000..0913ad7
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
@@ -0,0 +1,158 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * MockHTable Test.
+ *
+ * compare to real HTable.
+ */
+
+public class ITMockHTableTest extends HBaseMetadataTestCase {
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private static HTableInterface mock;
+ private static HTableInterface real;
+ private static String emailCF = "EMAIL";
+ private static String addressCF = "ADDRESS";
+ private static String emailCQ = "email";
+ private static String addressCQ = "address";
+ private static String tableName = "MockHTable-TEST";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ staticCreateTestMetadata();
+
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
+ real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
+ mock = new MockHTable(tableName, emailCF, addressCF);
+
+ setupDefaultData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ staticCleanupTestMetadata();
+ }
+
+ private static void setupDefaultData() throws IOException, InterruptedException {
+ Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
+ Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
+ Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
+ mutateAll(john, jane, me);
+ }
+
+ private static Put createPutForPerson(String name, String email, String address) {
+ Put person = new Put(name.getBytes());
+ person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
+ person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
+ return person;
+ }
+
+ private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
+ real.batch(Lists.newArrayList(rows));
+ mock.batch(Lists.newArrayList(rows));
+ }
+
+ @Test
+ public void test_get() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(false));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_get_with_filter() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+ get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
+
+ ));
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(true));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_exists() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_include_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_with_only_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(false));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_scan_with_filter() throws Exception {
+ Scan scan = new Scan();
+ scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ logger.debug("mock : {}", mockResult);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+
+ @Test
+ public void test_scan_for_pre_match() throws Exception {
+ Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
deleted file mode 100644
index 825fafc..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
-/**
- * MockHTable Test.
- *
- * compare to real HTable.
- */
-
-public class MockHTableTest extends HBaseMetadataTestCase {
- private final Logger logger = LoggerFactory.getLogger(this.getClass());
-
- private static HTableInterface mock;
- private static HTableInterface real;
- private static String emailCF = "EMAIL";
- private static String addressCF = "ADDRESS";
- private static String emailCQ = "email";
- private static String addressCQ = "address";
- private static String tableName = "MockHTable-TEST";
-
- @BeforeClass
- public static void setUp() throws Exception {
-
- staticCreateTestMetadata();
-
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
- real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
- mock = new MockHTable(tableName, emailCF, addressCF);
-
- setupDefaultData();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- staticCleanupTestMetadata();
- }
-
- private static void setupDefaultData() throws IOException, InterruptedException {
- Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
- Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
- Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
- mutateAll(john, jane, me);
- }
-
- private static Put createPutForPerson(String name, String email, String address) {
- Put person = new Put(name.getBytes());
- person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
- person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
- return person;
- }
-
- private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
- real.batch(Lists.newArrayList(rows));
- mock.batch(Lists.newArrayList(rows));
- }
-
- @Test
- public void test_get() throws Exception {
- Get get = new Get("John Doe".getBytes());
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(false));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_get_with_filter() throws Exception {
- Get get = new Get("John Doe".getBytes());
- get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
-
- ));
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(true));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_exists() throws IOException {
- Get get = new Get("John Doe".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_include_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_with_only_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(false));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_scan_with_filter() throws Exception {
- Scan scan = new Scan();
- scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- logger.debug("mock : {}", mockResult);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-
- @Test
- public void test_scan_for_pre_match() throws Exception {
- Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
deleted file mode 100644
index 25279c6..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class HiveTableReaderTest extends HBaseMetadataTestCase {
-
-
- @Test
- public void test() throws IOException {
- HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
- int rowNumber = 0;
- while (reader.next()) {
- String[] row = reader.getRow();
- Assert.assertEquals(9, row.length);
- //System.out.println(ArrayUtils.toString(row));
- rowNumber++;
- }
-
- reader.close();
- Assert.assertEquals(10000, rowNumber);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
new file mode 100644
index 0000000..8559f8b
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * This test case need the hive runtime; Please run it with sandbox;
+ * @author shaoshi
+ *
+ * It is in the exclude list of default profile in pom.xml
+ */
+public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
+
+
+ @Test
+ public void test() throws IOException {
+ HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
+ int rowNumber = 0;
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ Assert.assertEquals(9, row.length);
+ //System.out.println(ArrayUtils.toString(row));
+ rowNumber++;
+ }
+
+ reader.close();
+ Assert.assertEquals(10000, rowNumber);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
new file mode 100644
index 0000000..e59fcda
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTable;
+import org.apache.kylin.dict.lookup.SnapshotManager;
+import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.dict.lookup.TableReader;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author yangli9
+ *
+ */
+public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
+
+ SnapshotManager snapshotMgr;
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ snapshotMgr = SnapshotManager.getInstance(getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void basicTest() throws Exception {
+ String tableName = "EDW.TEST_SITES";
+ HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
+ TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
+ String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
+
+ snapshotMgr.wipeoutCache();
+
+ SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
+
+ // compare hive & snapshot
+ TableReader hiveReader = hiveTable.getReader();
+ TableReader snapshotReader = snapshot.getReader();
+
+ while (true) {
+ boolean hiveNext = hiveReader.next();
+ boolean snapshotNext = snapshotReader.next();
+ assertEquals(hiveNext, snapshotNext);
+
+ if (hiveNext == false)
+ break;
+
+ String[] hiveRow = hiveReader.getRow();
+ String[] snapshotRow = snapshotReader.getRow();
+ assertArrayEquals(hiveRow, snapshotRow);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index 0ae8e95..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.*;
-
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.dict.lookup.TableReader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
- SnapshotManager snapshotMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- snapshotMgr = SnapshotManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() throws Exception {
- String tableName = "EDW.TEST_SITES";
- HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
- snapshotMgr.wipeoutCache();
-
- SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
- // compare hive & snapshot
- TableReader hiveReader = hiveTable.getReader();
- TableReader snapshotReader = snapshot.getReader();
-
- while (true) {
- boolean hiveNext = hiveReader.next();
- boolean snapshotNext = snapshotReader.next();
- assertEquals(hiveNext, snapshotNext);
-
- if (hiveNext == false)
- break;
-
- String[] hiveRow = hiveReader.getRow();
- String[] snapshotRow = snapshotReader.getRow();
- assertArrayEquals(hiveRow, snapshotRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/Installation/Off Hadoop CLI Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation.md b/docs/Installation/Off Hadoop CLI Installation.md
index f83ed5d..a1877e3 100644
--- a/docs/Installation/Off Hadoop CLI Installation.md
+++ b/docs/Installation/Off Hadoop CLI Installation.md
@@ -53,7 +53,7 @@ An alternative to the host replacement is updating your `hosts` file to resolve
Run a end-to-end cube building test
- mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false
+ mvn test -Dtest=org.apache.kylin.job.ITBuildCubeWithEngineTest -DfailIfNoTests=false
Run other tests, the end-to-end cube building test is exclueded
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/MISC/FAQ on Kylin Installation and Usage.md
----------------------------------------------------------------------
diff --git a/docs/MISC/FAQ on Kylin Installation and Usage.md b/docs/MISC/FAQ on Kylin Installation and Usage.md
index a568614..6f24512 100644
--- a/docs/MISC/FAQ on Kylin Installation and Usage.md
+++ b/docs/MISC/FAQ on Kylin Installation and Usage.md
@@ -27,7 +27,7 @@ My Cluster is running on Cloudera Distribution CDH 5.2.0.
#### Unable to load a big cube as HTable, with java.lang.OutOfMemoryError: unable to create new native thread
HBase (as of writing) allocates one thread per region when bulk loading a HTable. Try reduce the number of regions of your cube by setting its "capacity" to "MEDIUM" or "LARGE". Also tweaks OS & JVM can allow more threads, for example see [this article](http://blog.egilh.com/2006/06/2811aspx.html).
-#### Failed to run BuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
+#### Failed to run ITBuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
User may get this error when first time run hbase client, please check the error trace to see whether there is an error saying couldn't access a folder like "/hadoop/hbase/local/jars"; If that folder doesn't exist, create it.
#### SUM(field) returns a negtive result while all the numbers in this field are > 0
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/docs/Test/Run Kylin test case with HBase Mini Cluster.md
----------------------------------------------------------------------
diff --git a/docs/Test/Run Kylin test case with HBase Mini Cluster.md b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
index 6088eb9..107da81 100644
--- a/docs/Test/Run Kylin test case with HBase Mini Cluster.md
+++ b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
@@ -8,7 +8,7 @@ mvn test -P sandbox
* Startup a HBase minicluster and update KylinConfig with the dynamic HBase configurations;
* Create Kylin metadata tables and import six example cube tables;
-* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running BuildCubeWithEngineTest）
+* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running ITBuildCubeWithEngineTest）
* After all test cases be completed, shutdown minicluster and cleanup KylinConfig cache;
### To ensure Mini cluster can run successfully, you need:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/jdbc/kylin_jdbc.log.2014-12-22
----------------------------------------------------------------------
diff --git a/jdbc/kylin_jdbc.log.2014-12-22 b/jdbc/kylin_jdbc.log.2014-12-22
new file mode 100644
index 0000000..7c5e001
--- /dev/null
+++ b/jdbc/kylin_jdbc.log.2014-12-22
@@ -0,0 +1,18 @@
+[main]:[2014-12-22 11:47:05,477][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,486][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,077][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,086][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@3add750e]
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 15:57:49,134][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,135][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,081][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,090][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
deleted file mode 100644
index 547a6e7..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class BuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
deleted file mode 100644
index dc42dc5..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class BuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42f124fb/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
deleted file mode 100644
index 9c0219a..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class BuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
-
- BuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}