You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/26 17:20:47 UTC
[01/32] incubator-kylin git commit: fix ci: new model API changed
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 ded2415b3 -> 74572f3ee
fix ci: new model API changed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/04eca53d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/04eca53d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/04eca53d
Branch: refs/heads/0.8.0
Commit: 04eca53dc62cae94dc7ac59348b499f58fbb2a21
Parents: 43ce8bc
Author: honma <ho...@ebay.com>
Authored: Mon May 25 10:33:23 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:11:52 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeControllerTest.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/04eca53d/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 1ec27d5..d2a948b 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
@@ -66,7 +65,7 @@ public class CubeControllerTest extends ServiceTestBase {
CubeDesc[] cubes = (CubeDesc[]) cubeDescController.getCube("test_kylin_cube_with_slr_ready");
Assert.assertNotNull(cubes);
Assert.assertNotNull(cubeController.getSql("test_kylin_cube_with_slr_ready", "20130331080000_20131212080000"));
- Assert.assertNotNull(cubeController.getCubes(null, null, null,0, 5));
+ Assert.assertNotNull(cubeController.getCubes(null, null, null, 0, 5));
CubeDesc cube = cubes[0];
CubeDesc newCube = new CubeDesc();
@@ -87,8 +86,6 @@ public class CubeControllerTest extends ServiceTestBase {
newCube.setConfig(cube.getConfig());
newCube.setRowkey(cube.getRowkey());
- String newModelName = newCubeName + "_model_desc";
- newCube.getModel().setName(newModelName);//generate a random model
newCube.getModel().setLastModified(0);
ObjectMapper cubeDescMapper = new ObjectMapper();
@@ -103,13 +100,12 @@ public class CubeControllerTest extends ServiceTestBase {
cubeRequest.setCubeDescData(cubeDescWriter.toString());
cubeRequest = cubeController.saveCubeDesc(cubeRequest);
-
List<String> notifyList = Lists.newArrayList();
notifyList.add("john@example.com");
cubeController.updateNotifyList(newCubeName, notifyList);
cubeController.updateCubeCost(newCubeName, 80);
- List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, "default",null, 1, 0);
+ List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, cube.getModelName(),"default", 1, 0);
CubeInstance cubeInstance = cubeInstances.get(0);
Assert.assertTrue(cubeInstance.getDescriptor().getNotifyList().contains("john@example.com"));
[29/32] incubator-kylin git commit: KYLIN-697 fix minicluster profile
Posted by ma...@apache.org.
KYLIN-697 fix minicluster profile
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d647d5a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d647d5a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d647d5a5
Branch: refs/heads/0.8.0
Commit: d647d5a546b6466f283bcedb7e5f37859980a732
Parents: 7749479
Author: honma <ho...@ebay.com>
Authored: Tue May 26 23:03:28 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:28 2015 +0800
----------------------------------------------------------------------
pom.xml | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d647d5a5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 21fbed8..adfda4c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -564,13 +564,18 @@
<configuration>
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
- <exclude>**/IT*.java</exclude>
<!--Build cube/II need to be run separately-->
+ <exclude>**/BuildCube*.java</exclude>
+ <exclude>**/BuildII*.java</exclude>
+
+ <!--minicluster does not have kafka-->
+ <exclude>**/ITKafka*.java</exclude>
<!--minicluster does not have hive-->
<exclude>**/ITSnapshotManagerTest.java</exclude>
<exclude>**/ITHiveTableReaderTest.java</exclude>
<exclude>**/ITTableControllerTest.java</exclude>
+
</excludes>
<systemProperties>
<property>
@@ -605,6 +610,8 @@
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
<exclude>**/IT*.java</exclude>
+
+ <!--Build cube/II need to be run separately-->
<exclude>**/BuildCube*.java</exclude>
<exclude>**/BuildII*.java</exclude>
</excludes>
[10/32] incubator-kylin git commit: KYLIN-693 character all build
cube/ii test cases
Posted by ma...@apache.org.
KYLIN-693 character all build cube/ii test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f280df69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f280df69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f280df69
Branch: refs/heads/0.8.0
Commit: f280df6917f66743497b1689c3b12fac0dbf5f02
Parents: 4e35d9f
Author: honma <ho...@ebay.com>
Authored: Mon May 25 14:15:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:17:16 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/BuildCubeWithStreamTest.java | 224 +++++++++++++++
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 +++++++++++++++++
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 ---------------
.../kylin/job/ITBuildIIWithEngineTest.java | 254 -----------------
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ------------------
pom.xml | 162 ++++++-----
9 files changed, 1122 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
new file mode 100644
index 0000000..547a6e7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.cube.CubingJob;
+import org.apache.kylin.job.cube.CubingJobBuilder;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class BuildCubeWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+
+ private CubeManager cubeManager;
+
+ private DefaultScheduler scheduler;
+
+ protected ExecutableManager jobService;
+
+ private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if(jobService.getJob(jobId) instanceof CubingJob){
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
+ testInner();
+ testLeft();
+ }
+
+ private void testInner() throws Exception {
+ String[] testCase = new String[]{
+ "testInnerJoinCube",
+ "testInnerJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void testLeft() throws Exception {
+ String[] testCase = new String[]{
+ "testLeftJoinCube",
+ "testLeftJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void runTestAndAssertSucceed(String[] testCase) throws Exception {
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ try {
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex);
+ throw ex;
+ }
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube2() throws Exception {
+ clearSegment("test_kylin_cube_with_slr_empty");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
+
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ // this cube's start date is 0, end date is 20501112000000
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+
+
+ // this cube doesn't support incremental build, always do full build
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube2() throws Exception {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ List<String> result = Lists.newArrayList();
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
+ // this cube's start date is 0, end date is 20120601000000
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2012-06-01").getTime();
+
+ clearSegment(cubeName);
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+
+ // then submit an append job, start date is 20120601000000, end
+ // date is 20220101000000
+ dateStart = f.parse("2012-06-01").getTime();
+ dateEnd = f.parse("2022-01-01").getTime();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ clearSegment(cubeName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2050-11-12").getTime();
+
+ // this cube's start date is 0, end date is 20501112000000
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube,true);
+ }
+
+
+ private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
+ CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
+ CubingJob job = cubingJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
new file mode 100644
index 0000000..dc42dc5
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -0,0 +1,224 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
+import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ *
+ * This class is going to be deleted
+ */
+@Ignore("For dev testing")
+public class BuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ // cube.getSegments().clear();
+ // cubeManager.updateCube(cube);
+
+ CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
+ Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+
+//
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = cubeSegment.getDictionary(col);
+ if (dict == null) {
+ throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
+ }
+ logger.info("Dictionary for " + col + " was put into dictionary map.");
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
+ }
+ }
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
+ tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
+ logger.info("intermediate table name:" + tableName);
+
+
+ ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
+
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ Future<?> future = executorService.submit(cubeBuilder);
+
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ List<String> row;
+ int counter = 0;
+ while (reader.next()) {
+ row = reader.getRowAsList();
+ queue.put(row);
+ counter++;
+ if(counter == 200000)
+ break;
+ }
+ queue.put(new ArrayList<String>(0));
+ reader.close();
+
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ throw new IOException("Failed to build cube ", e);
+ }
+
+ logger.info("stream build finished");
+ }
+
+
+ private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ if (input == null)
+ return null;
+ return input.getBytes();
+ }
+ }));
+
+ logger.info("Building dictionary for " + col);
+ dictionaryMap.put(col, dict);
+ }
+ }
+ }
+
+ }
+
+
+ class ConsoleGTRecordWriter implements IGTRecordWriter {
+
+ boolean verbose = false;
+
+ @Override
+ public void write(Long cuboidId, GTRecord record) throws IOException {
+ if (verbose)
+ System.out.println(record.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
new file mode 100644
index 0000000..9c0219a
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.invertedindex.IIJob;
+import org.apache.kylin.job.invertedindex.IIJobBuilder;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author shaoshi
+ */
+public class BuildIIWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+ private IIManager iiManager;
+
+ private DefaultScheduler scheduler;
+ protected ExecutableManager jobService;
+
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
+
+ private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ //DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if (jobService.getJob(jobId) instanceof IIJob) {
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : TEST_II_INSTANCES) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii, true);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+
+ for (String iiInstance : TEST_II_INSTANCES) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii, true);
+ }
+ }
+ backup();
+ }
+
+ @Test
+ @Ignore
+ public void testBuildII() throws Exception {
+
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildIIWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ protected List<String> buildIIInnerJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[0]);
+ }
+
+ protected List<String> buildIILeftJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[1]);
+ }
+
+ protected List<String> buildII(String iiName) throws Exception {
+ clearSegment(iiName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(iiName, date1, date2));
+ return result;
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance, true);
+ IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
+ IIJob job = iiJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ public static void main(String[] args) throws Exception {
+ BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
+
+ BuildIIWithEngineTest.beforeClass();
+ instance.before();
+ instance.testBuildII();
+ instance.after();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
new file mode 100644
index 0000000..6bceab7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class BuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
deleted file mode 100644
index e99bd01..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class ITBuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(ITBuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
deleted file mode 100644
index 8a26397..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class ITBuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
deleted file mode 100644
index 23f97d0..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class ITBuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(ITBuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- ITBuildIIWithEngineTest instance = new ITBuildIIWithEngineTest();
-
- ITBuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
deleted file mode 100644
index cfb7bdb..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.streaming.StreamMessage;
-import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
-import static org.junit.Assert.fail;
-
-/**
- */
-public class ITBuildIIWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
-
- private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
- private IIManager iiManager;
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- iiManager = IIManager.getInstance(kylinConfig);
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : II_NAME) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- backup();
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
- IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
- JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
- final String uuid = UUID.randomUUID().toString();
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- logger.info(step.getCmd());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
- kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
- return intermediateTableDesc.getTableName(uuid);
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private IISegment createSegment(String iiName) throws Exception {
- clearSegment(iiName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- return buildSegment(iiName, date1, date2);
- }
-
- private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
- return segment;
- }
-
- private void buildII(String iiName) throws Exception {
- final IIDesc desc = iiManager.getII(iiName).getDescriptor();
- final String tableName = createIntermediateTable(desc, kylinConfig);
- logger.info("intermediate table name:" + tableName);
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- for (TblColRef tblColRef : tblColRefs) {
- if (desc.isMetricsCol(tblColRef)) {
- logger.info("matrix:" + tblColRef.getName());
- } else {
- logger.info("measure:" + tblColRef.getName());
- }
- }
- LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
- final IISegment segment = createSegment(iiName);
- String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
- ToolRunner.run(new IICreateHTableJob(), args);
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
-
- List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
- int count = sorted.size();
- for (String[] row : sorted) {
- logger.info("another row: " + StringUtils.join(row, ","));
- queue.put(parse(row));
- }
-
- reader.close();
- logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
- queue.put(StreamMessage.EOF);
- final Future<?> future = executorService.submit(streamBuilder);
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- fail("stream build failed");
- }
-
- logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
- }
-
- @Test
- public void test() throws Exception {
- for (String iiName : II_NAME) {
- buildII(iiName);
- IIInstance ii = iiManager.getII(iiName);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- private StreamMessage parse(String[] row) {
- return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
- }
-
- private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
- List<String[]> unsorted = Lists.newArrayList();
- while (reader.next()) {
- unsorted.add(reader.getRow());
- }
- Collections.sort(unsorted, new Comparator<String[]>() {
- @Override
- public int compare(String[] o1, String[] o2) {
- long t1 = DateFormat.stringToMillis(o1[tsCol]);
- long t2 = DateFormat.stringToMillis(o2[tsCol]);
- return Long.compare(t1, t2);
- }
- });
- return unsorted;
- }
-
-}
[07/32] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
deleted file mode 100644
index 6bceab7..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.streaming.StreamMessage;
-import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
-import static org.junit.Assert.fail;
-
-/**
- */
-public class BuildIIWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
-
- private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
- private IIManager iiManager;
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- iiManager = IIManager.getInstance(kylinConfig);
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : II_NAME) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- backup();
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
- IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
- JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
- final String uuid = UUID.randomUUID().toString();
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- logger.info(step.getCmd());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
- kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
- return intermediateTableDesc.getTableName(uuid);
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private IISegment createSegment(String iiName) throws Exception {
- clearSegment(iiName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- return buildSegment(iiName, date1, date2);
- }
-
- private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
- return segment;
- }
-
- private void buildII(String iiName) throws Exception {
- final IIDesc desc = iiManager.getII(iiName).getDescriptor();
- final String tableName = createIntermediateTable(desc, kylinConfig);
- logger.info("intermediate table name:" + tableName);
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- for (TblColRef tblColRef : tblColRefs) {
- if (desc.isMetricsCol(tblColRef)) {
- logger.info("matrix:" + tblColRef.getName());
- } else {
- logger.info("measure:" + tblColRef.getName());
- }
- }
- LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
- final IISegment segment = createSegment(iiName);
- String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
- ToolRunner.run(new IICreateHTableJob(), args);
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
-
- List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
- int count = sorted.size();
- for (String[] row : sorted) {
- logger.info("another row: " + StringUtils.join(row, ","));
- queue.put(parse(row));
- }
-
- reader.close();
- logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
- queue.put(StreamMessage.EOF);
- final Future<?> future = executorService.submit(streamBuilder);
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- fail("stream build failed");
- }
-
- logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
- }
-
- @Test
- public void test() throws Exception {
- for (String iiName : II_NAME) {
- buildII(iiName);
- IIInstance ii = iiManager.getII(iiName);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- private StreamMessage parse(String[] row) {
- return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
- }
-
- private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
- List<String[]> unsorted = Lists.newArrayList();
- while (reader.next()) {
- unsorted.add(reader.getRow());
- }
- Collections.sort(unsorted, new Comparator<String[]>() {
- @Override
- public int compare(String[] o1, String[] o2) {
- long t1 = DateFormat.stringToMillis(o1[tsCol]);
- long t2 = DateFormat.stringToMillis(o2[tsCol]);
- return Long.compare(t1, t2);
- }
- });
- return unsorted;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
deleted file mode 100644
index 6aaeaa2..0000000
--- a/job/src/test/java/org/apache/kylin/job/IIStreamBuilderTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- */
-public class IIStreamBuilderTest {
-
- private static final Logger logger = LoggerFactory.getLogger(IIStreamBuilderTest.class);
-
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
-// backup();
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
- }
-
- @Test
- public void test() throws Exception {
- final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
- bootstrap.start("eagle", 0);
- Thread.sleep(30 * 60 * 1000);
- logger.info("time is up, stop streaming");
- bootstrap.stop();
- Thread.sleep(5 * 1000);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
new file mode 100644
index 0000000..e99bd01
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.cube.CubingJob;
+import org.apache.kylin.job.cube.CubingJobBuilder;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class ITBuildCubeWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+
+ private CubeManager cubeManager;
+
+ private DefaultScheduler scheduler;
+
+ protected ExecutableManager jobService;
+
+ private static final Log logger = LogFactory.getLog(ITBuildCubeWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if(jobService.getJob(jobId) instanceof CubingJob){
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
+ testInner();
+ testLeft();
+ }
+
+ private void testInner() throws Exception {
+ String[] testCase = new String[]{
+ "testInnerJoinCube",
+ "testInnerJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void testLeft() throws Exception {
+ String[] testCase = new String[]{
+ "testLeftJoinCube",
+ "testLeftJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void runTestAndAssertSucceed(String[] testCase) throws Exception {
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ try {
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex);
+ throw ex;
+ }
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = ITBuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(ITBuildCubeWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube2() throws Exception {
+ clearSegment("test_kylin_cube_with_slr_empty");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
+
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ // this cube's start date is 0, end date is 20501112000000
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+
+
+ // this cube doesn't support incremental build, always do full build
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube2() throws Exception {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ List<String> result = Lists.newArrayList();
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
+ // this cube's start date is 0, end date is 20120601000000
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2012-06-01").getTime();
+
+ clearSegment(cubeName);
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+
+ // then submit an append job, start date is 20120601000000, end
+ // date is 20220101000000
+ dateStart = f.parse("2012-06-01").getTime();
+ dateEnd = f.parse("2022-01-01").getTime();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ clearSegment(cubeName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2050-11-12").getTime();
+
+ // this cube's start date is 0, end date is 20501112000000
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube,true);
+ }
+
+
+ private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
+ CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
+ CubingJob job = cubingJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
new file mode 100644
index 0000000..8a26397
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
@@ -0,0 +1,224 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
+import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ *
+ * This class is going to be deleted
+ */
+@Ignore("For dev testing")
+public class ITBuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITBuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ // cube.getSegments().clear();
+ // cubeManager.updateCube(cube);
+
+ CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
+ Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+
+//
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = cubeSegment.getDictionary(col);
+ if (dict == null) {
+ throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
+ }
+ logger.info("Dictionary for " + col + " was put into dictionary map.");
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
+ }
+ }
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
+ tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
+ logger.info("intermediate table name:" + tableName);
+
+
+ ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
+
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ Future<?> future = executorService.submit(cubeBuilder);
+
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ List<String> row;
+ int counter = 0;
+ while (reader.next()) {
+ row = reader.getRowAsList();
+ queue.put(row);
+ counter++;
+ if(counter == 200000)
+ break;
+ }
+ queue.put(new ArrayList<String>(0));
+ reader.close();
+
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ throw new IOException("Failed to build cube ", e);
+ }
+
+ logger.info("stream build finished");
+ }
+
+
+ private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ if (input == null)
+ return null;
+ return input.getBytes();
+ }
+ }));
+
+ logger.info("Building dictionary for " + col);
+ dictionaryMap.put(col, dict);
+ }
+ }
+ }
+
+ }
+
+
+ class ConsoleGTRecordWriter implements IGTRecordWriter {
+
+ boolean verbose = false;
+
+ @Override
+ public void write(Long cuboidId, GTRecord record) throws IOException {
+ if (verbose)
+ System.out.println(record.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
new file mode 100644
index 0000000..23f97d0
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.invertedindex.IIJob;
+import org.apache.kylin.job.invertedindex.IIJobBuilder;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author shaoshi
+ */
+public class ITBuildIIWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+ private IIManager iiManager;
+
+ private DefaultScheduler scheduler;
+ protected ExecutableManager jobService;
+
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
+
+ private static final Log logger = LogFactory.getLog(ITBuildIIWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ //DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if (jobService.getJob(jobId) instanceof IIJob) {
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : TEST_II_INSTANCES) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii, true);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+
+ for (String iiInstance : TEST_II_INSTANCES) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii, true);
+ }
+ }
+ backup();
+ }
+
+ @Test
+ @Ignore
+ public void testBuildII() throws Exception {
+
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = ITBuildIIWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(ITBuildIIWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ protected List<String> buildIIInnerJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[0]);
+ }
+
+ protected List<String> buildIILeftJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[1]);
+ }
+
+ protected List<String> buildII(String iiName) throws Exception {
+ clearSegment(iiName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(iiName, date1, date2));
+ return result;
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance, true);
+ IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
+ IIJob job = iiJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ public static void main(String[] args) throws Exception {
+ ITBuildIIWithEngineTest instance = new ITBuildIIWithEngineTest();
+
+ ITBuildIIWithEngineTest.beforeClass();
+ instance.before();
+ instance.testBuildII();
+ instance.after();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
new file mode 100644
index 0000000..cfb7bdb
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class ITBuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
new file mode 100644
index 0000000..2edf457
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
@@ -0,0 +1,108 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.streaming.StreamingBootstrap;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ */
+public class ITIIStreamBuilderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITIIStreamBuilderTest.class);
+
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+// backup();
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = {"--delete", "true"};
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+ }
+
+ @Test
+ public void test() throws Exception {
+ final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
+ bootstrap.start("eagle", 0);
+ Thread.sleep(30 * 60 * 1000);
+ logger.info("time is up, stop streaming");
+ bootstrap.stop();
+ Thread.sleep(5 * 1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
deleted file mode 100644
index 22afb1d..0000000
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/DefaultSchedulerTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.impl.threadpool;
-
-import static org.junit.Assert.*;
-
-import org.apache.kylin.job.SelfStopExecutable;
-import org.junit.Test;
-
-import org.apache.kylin.job.BaseTestExecutable;
-import org.apache.kylin.job.ErrorTestExecutable;
-import org.apache.kylin.job.FailedTestExecutable;
-import org.apache.kylin.job.SucceedTestExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-
-/**
- */
-public class DefaultSchedulerTest extends BaseSchedulerTest {
-
- @Test
- public void testSingleTaskJob() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- }
-
- @Test
- public void testSucceed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState());
- }
- @Test
- public void testSucceedAndFailed() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SucceedTestExecutable();
- BaseTestExecutable task2 = new FailedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState());
- }
- @Test
- public void testSucceedAndError() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new ErrorTestExecutable();
- BaseTestExecutable task2 = new SucceedTestExecutable();
- job.addTask(task1);
- job.addTask(task2);
- jobService.addJob(job);
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState());
- assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState());
- }
-
- @Test
- public void testDiscard() throws Exception {
- DefaultChainedExecutable job = new DefaultChainedExecutable();
- BaseTestExecutable task1 = new SelfStopExecutable();
- job.addTask(task1);
- jobService.addJob(job);
- waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
- jobService.discardJob(job.getId());
- waitForJobFinish(job.getId());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
- assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
- Thread.sleep(5000);
- System.out.println(job);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
new file mode 100644
index 0000000..bf4a1dc
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/ITDefaultSchedulerTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.impl.threadpool;
+
+import org.apache.kylin.job.*;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ */
+public class ITDefaultSchedulerTest extends BaseSchedulerTest {
+
+ @Test
+ public void testSingleTaskJob() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ job.addTask(task1);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ }
+
+ @Test
+ public void testSucceed() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ BaseTestExecutable task2 = new SucceedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task2.getId()).getState());
+ }
+ @Test
+ public void testSucceedAndFailed() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SucceedTestExecutable();
+ BaseTestExecutable task2 = new FailedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(task2.getId()).getState());
+ }
+ @Test
+ public void testSucceedAndError() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new ErrorTestExecutable();
+ BaseTestExecutable task2 = new SucceedTestExecutable();
+ job.addTask(task1);
+ job.addTask(task2);
+ jobService.addJob(job);
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.ERROR, jobService.getOutput(task1.getId()).getState());
+ assertEquals(ExecutableState.READY, jobService.getOutput(task2.getId()).getState());
+ }
+
+ @Test
+ public void testDiscard() throws Exception {
+ DefaultChainedExecutable job = new DefaultChainedExecutable();
+ BaseTestExecutable task1 = new SelfStopExecutable();
+ job.addTask(task1);
+ jobService.addJob(job);
+ waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);
+ jobService.discardJob(job.getId());
+ waitForJobFinish(job.getId());
+ assertEquals(ExecutableState.DISCARDED, jobService.getOutput(job.getId()).getState());
+ assertEquals(ExecutableState.DISCARDED, jobService.getOutput(task1.getId()).getState());
+ Thread.sleep(5000);
+ System.out.println(job);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java b/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
deleted file mode 100644
index 0534f0f..0000000
--- a/metadata/src/test/java/org/apache/kylin/metadata/tool/HiveSourceTableLoaderTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata.tool;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.util.HiveSourceTableLoader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HiveSourceTableLoaderTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- super.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- super.cleanupTestMetadata();
- }
-
- @Test
- public void test() throws IOException {
- if (!useSandbox())
- return;
-
- KylinConfig config = getTestConfig();
- String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
- Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
-
- assertTrue(loaded.size() == toLoad.length);
- for (String str : toLoad)
- assertTrue(loaded.contains(str));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java b/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
new file mode 100644
index 0000000..f357855
--- /dev/null
+++ b/metadata/src/test/java/org/apache/kylin/metadata/tool/ITHiveSourceTableLoaderTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.tool;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.metadata.util.HiveSourceTableLoader;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ super.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ super.cleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws IOException {
+ if (!useSandbox())
+ return;
+
+ KylinConfig config = getTestConfig();
+ String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
+ Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
+
+ assertTrue(loaded.size() == toLoad.length);
+ for (String str : toLoad)
+ assertTrue(loaded.contains(str));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2e9b4f9..a72b59d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -562,19 +562,13 @@
<configuration>
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
+ <exclude>**/IT*.java</exclude>
<!--Build cube/II need to be run separately-->
- <exclude>**/BuildCubeWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithStreamTest.java</exclude>
- <exclude>**/IIStreamBuilderTest.java</exclude>
<!--minicluster does not have hive-->
- <exclude>**/SnapshotManagerTest.java</exclude>
- <exclude>**/HiveTableReaderTest.java</exclude>
- <exclude>**/TableControllerTest.java</exclude>
-
- <exclude>**/Kafka*Test.java</exclude>
- <exclude>**/RequesterTest.java</exclude>
+ <exclude>**/ITSnapshotManagerTest.java</exclude>
+ <exclude>**/ITHiveTableReaderTest.java</exclude>
+ <exclude>**/ITTableControllerTest.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -608,15 +602,7 @@
<configuration>
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
- <!--Build cube/II need to be run separately-->
- <exclude>**/BuildCubeWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithEngineTest.java</exclude>
- <exclude>**/BuildIIWithStreamTest.java</exclude>
- <exclude>**/IIStreamBuilderTest.java</exclude>
-
- <exclude>**/Kafka*Test.java</exclude>
- <exclude>**/RequesterTest.java</exclude>
- <exclude>**/InMemCubeBuilderBenchmarkTest.java</exclude>
+ <exclude>**/IT*.java</exclude>
</excludes>
<systemProperties>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
deleted file mode 100644
index 9e8e92b..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/CombinationTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
-
-/**
- */
-@RunWith(Parameterized.class)
-public class CombinationTest extends KylinQueryTest {
-
- @BeforeClass
- public static void setUp() throws SQLException {
- System.out.println("setUp in CombinationTest");
- }
-
- @AfterClass
- public static void tearDown() {
- clean();
- }
-
- /**
- * return all config combinations, where first setting specifies join type
- * (inner or left), and the second setting specifies whether to force using
- * coprocessors(on, off or unset).
- */
- @Parameterized.Parameters
- public static Collection<Object[]> configs() {
-// return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
- return Arrays.asList(new Object[][]{{"inner", "on"}, {"left", "on"}});
- }
-
- public CombinationTest(String joinType, String coprocessorToggle) throws Exception {
-
- KylinQueryTest.clean();
-
- KylinQueryTest.joinType = joinType;
- KylinQueryTest.setupAll();
-
- if (coprocessorToggle.equals("on")) {
- ObserverEnabler.forceCoprocessorOn();
- } else if (coprocessorToggle.equals("off")) {
- ObserverEnabler.forceCoprocessorOff();
- } else if (coprocessorToggle.equals("unset")) {
- // unset
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
deleted file mode 100644
index 185b22e..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-
-/**
- */
-@RunWith(Parameterized.class)
-public class IIQueryTest extends KylinQueryTest {
- @BeforeClass
- public static void setUp() throws Exception {
-
- // give II higher priority than other realizations
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 0);
- priorities.put(RealizationType.CUBE, 1);
- priorities.put(RealizationType.HYBRID, 1);
- RealizationPriorityRule.setPriorities(priorities);
-
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- KylinQueryTest.tearDown();//invoke super class
-
- Map<RealizationType, Integer> priorities = Maps.newHashMap();
- priorities.put(RealizationType.INVERTED_INDEX, 1);
- priorities.put(RealizationType.CUBE, 0);
- priorities.put(RealizationType.HYBRID, 0);
- RealizationPriorityRule.setPriorities(priorities);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> configs() {
- return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
- }
-
- public IIQueryTest(String joinType) throws Exception {
-
- KylinQueryTest.clean();
-
- KylinQueryTest.joinType = joinType;
- KylinQueryTest.setupAll();
-
- }
-
- @Test
- public void testSingleRunQuery() throws Exception {
- super.testSingleRunQuery();
- }
-
- @Test
- public void testDetailedQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_ii", null, true);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
new file mode 100644
index 0000000..a38e85c
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ */
+@RunWith(Parameterized.class)
+public class ITCombinationTest extends ITKylinQueryTest {
+
+ @BeforeClass
+ public static void setUp() throws SQLException {
+ System.out.println("setUp in ITCombinationTest");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ clean();
+ }
+
+ /**
+ * return all config combinations, where first setting specifies join type
+ * (inner or left), and the second setting specifies whether to force using
+ * coprocessors(on, off or unset).
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+// return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
+ return Arrays.asList(new Object[][]{{"inner", "on"}, {"left", "on"}});
+ }
+
+ public ITCombinationTest(String joinType, String coprocessorToggle) throws Exception {
+
+ ITKylinQueryTest.clean();
+
+ ITKylinQueryTest.joinType = joinType;
+ ITKylinQueryTest.setupAll();
+
+ if (coprocessorToggle.equals("on")) {
+ ObserverEnabler.forceCoprocessorOn();
+ } else if (coprocessorToggle.equals("off")) {
+ ObserverEnabler.forceCoprocessorOff();
+ } else if (coprocessorToggle.equals("unset")) {
+ // unset
+ }
+ }
+}
[30/32] incubator-kylin git commit: KYLIN-697 totally avoid ZK in
localmeta test cases
Posted by ma...@apache.org.
KYLIN-697 totally avoid ZK in localmeta test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5b6960ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5b6960ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5b6960ee
Branch: refs/heads/0.8.0
Commit: 5b6960eed1105750de3c751dfb1e78b428a7a3d0
Parents: 0c5d549
Author: honma <ho...@ebay.com>
Authored: Tue May 26 20:00:19 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:28 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/lock/JobLock.java | 10 +
.../apache/kylin/common/lock/MockJobLock.java | 16 ++
.../kylin/common/lock/ZookeeperJobLock.java | 81 ++++++
.../common/util/AbstractKylinTestCase.java | 11 +-
.../common/util/HBaseMetadataTestCase.java | 10 +
.../common/util/LocalFileMetadataTestCase.java | 11 +-
.../kylin/metadata/MetadataUpgradeTest.java | 17 +-
.../java/org/apache/kylin/job/Scheduler.java | 3 +-
.../job/impl/threadpool/DefaultScheduler.java | 76 +-----
.../kylin/job/BuildCubeWithEngineTest.java | 33 ++-
.../apache/kylin/job/BuildIIWithEngineTest.java | 3 +-
.../job/impl/threadpool/BaseSchedulerTest.java | 3 +-
.../kylin/rest/controller/JobController.java | 6 +-
.../kylin/rest/security/AclHBaseStorage.java | 25 ++
.../rest/security/MockAclHBaseStorage.java | 45 ++++
.../rest/security/RealAclHBaseStorage.java | 51 ++++
.../apache/kylin/rest/service/AclService.java | 2 +-
.../apache/kylin/rest/service/UserService.java | 2 +-
.../apache/kylin/rest/util/AclHBaseStorage.java | 25 --
.../kylin/rest/util/MockAclHBaseStorage.java | 45 ----
.../kylin/rest/util/RealAclHBaseStorage.java | 51 ----
.../src/main/resources/applicationContext.xml | 244 ++++++++++---------
.../kylin/rest/service/CacheServiceTest.java | 2 +-
23 files changed, 413 insertions(+), 359 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/JobLock.java b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
new file mode 100644
index 0000000..5293385
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/JobLock.java
@@ -0,0 +1,10 @@
+package org.apache.kylin.common.lock;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/26/15.
+ */
+public interface JobLock {
+ boolean lock();
+
+ void unlock();
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
new file mode 100644
index 0000000..9285fd2
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/MockJobLock.java
@@ -0,0 +1,16 @@
+package org.apache.kylin.common.lock;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/26/15.
+ */
+public class MockJobLock implements JobLock {
+ @Override
+ public boolean lock() {
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
new file mode 100644
index 0000000..7a755a3
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
@@ -0,0 +1,81 @@
+package org.apache.kylin.common.lock;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/26/15.
+ */
+public class ZookeeperJobLock implements JobLock {
+ private Logger logger = LoggerFactory.getLogger(ZookeeperJobLock.class);
+
+ private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+ private InterProcessMutex sharedLock;
+ private CuratorFramework zkClient;
+
+ @Override
+ public boolean lock() {
+ String ZKConnectString = getZKConnectString();
+ if (StringUtils.isEmpty(ZKConnectString)) {
+ throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
+ }
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
+ this.zkClient.start();
+ this.sharedLock = new InterProcessMutex(zkClient, schedulerId());
+ boolean hasLock = false;
+ try {
+ hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ logger.warn("error acquire lock", e);
+ }
+ if (!hasLock) {
+ logger.warn("fail to acquire lock, scheduler has not been started");
+ zkClient.close();
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void unlock() {
+ releaseLock();
+ }
+
+ private String getZKConnectString() {
+ Configuration conf = HadoopUtil.newHBaseConfiguration(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ }
+
+ private void releaseLock() {
+ try {
+ if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
+ // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
+ if (zkClient.checkExists().forPath(schedulerId()) != null) {
+ zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(schedulerId());
+ }
+ }
+ } catch (Exception e) {
+ logger.error("error release lock:" + schedulerId());
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String schedulerId() {
+ return ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 36a9287..3fd82b3 100644
--- a/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -18,10 +18,10 @@
package org.apache.kylin.common.util;
-import java.lang.reflect.Method;
-
import org.apache.kylin.common.KylinConfig;
+import java.lang.reflect.Method;
+
/**
* @author ysong1
*
@@ -46,14 +46,7 @@ public abstract class AbstractKylinTestCase {
return KylinConfig.getInstanceFromEnv();
}
- public static void staticCreateTestMetadata(String kylinConfigFolder) {
- KylinConfig.destoryInstance();
-
- if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
- System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
-
- }
public static void staticCleanupTestMetadata() {
cleanupCache();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
index 90970e5..6473b3d 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HBaseMetadataTestCase.java
@@ -18,6 +18,8 @@
package org.apache.kylin.common.util;
+import org.apache.kylin.common.KylinConfig;
+
import java.io.File;
/**
@@ -54,6 +56,14 @@ public class HBaseMetadataTestCase extends AbstractKylinTestCase {
}
}
+ public static void staticCreateTestMetadata(String kylinConfigFolder) {
+
+ KylinConfig.destoryInstance();
+
+ if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
+ System.setProperty(KylinConfig.KYLIN_CONF, kylinConfigFolder);
+
+ }
public static boolean useSandbox() {
String useSandbox = System.getProperty("useSandbox");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java b/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index 60db6eb..1bd218e 100644
--- a/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ b/common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -18,14 +18,13 @@
package org.apache.kylin.common.util;
-import java.io.File;
-import java.io.IOException;
-
import org.apache.commons.io.FileUtils;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import java.io.File;
+import java.io.IOException;
+
/**
* @author ysong1
*/
@@ -33,10 +32,10 @@ public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
@Override
public void createTestMetadata() {
- createTestMetadata(LOCALMETA_TEST_DATA);
+ staticCreateTestMetadata(LOCALMETA_TEST_DATA);
}
- public static void createTestMetadata(String testDataFolder) {
+ public static void staticCreateTestMetadata(String testDataFolder) {
KylinConfig.destoryInstance();
String tempTestMetadataUrl = "../examples/test_metadata";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
index 4b1459c..17db72c 100644
--- a/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ b/cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
@@ -18,16 +18,6 @@
package org.apache.kylin.metadata;
-import java.util.List;
-
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -36,6 +26,11 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.*;
+
+import java.util.List;
/**
* Test the data model upgrade
@@ -47,7 +42,7 @@ public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
- createTestMetadata(LOCALMETA_TEST_DATA_V1);
+ staticCreateTestMetadata(LOCALMETA_TEST_DATA_V1);
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/job/src/main/java/org/apache/kylin/job/Scheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/Scheduler.java b/job/src/main/java/org/apache/kylin/job/Scheduler.java
index 592b274..fe6fe95 100644
--- a/job/src/main/java/org/apache/kylin/job/Scheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/Scheduler.java
@@ -18,6 +18,7 @@
package org.apache.kylin.job;
+import org.apache.kylin.common.lock.JobLock;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.SchedulerException;
import org.apache.kylin.job.execution.Executable;
@@ -26,7 +27,7 @@ import org.apache.kylin.job.execution.Executable;
*/
public interface Scheduler<T extends Executable> {
- void init(JobEngineConfig jobEngineConfig) throws SchedulerException;
+ void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException;
void shutdown() throws SchedulerException;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 2acba4e..bac4227 100644
--- a/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -18,31 +18,17 @@
package org.apache.kylin.job.impl.threadpool;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.RetryPolicy;
+import com.google.common.collect.Maps;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.imps.CuratorFrameworkState;
-import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.framework.state.ConnectionState;
import org.apache.curator.framework.state.ConnectionStateListener;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
+import org.apache.kylin.common.lock.JobLock;
import org.apache.kylin.job.Scheduler;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.SchedulerException;
+import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.Executable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.Output;
@@ -50,14 +36,13 @@ import org.apache.kylin.job.manager.ExecutableManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
-import org.apache.kylin.common.util.HadoopUtil;
+import java.util.Map;
+import java.util.concurrent.*;
/**
*/
public class DefaultScheduler implements Scheduler<AbstractExecutable>, ConnectionStateListener {
- private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
private ExecutableManager executableManager;
private ScheduledExecutorService fetcherPool;
@@ -67,9 +52,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
private Logger logger = LoggerFactory.getLogger(DefaultScheduler.class);
private volatile boolean initialized = false;
private volatile boolean hasStarted = false;
- private CuratorFramework zkClient;
private JobEngineConfig jobEngineConfig;
- private InterProcessMutex sharedLock;
private static final DefaultScheduler INSTANCE = new DefaultScheduler();
@@ -139,29 +122,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
}
}
- private void releaseLock() {
- try {
- if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
- // client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
- if (zkClient.checkExists().forPath(schedulerId()) != null) {
- zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(schedulerId());
- }
- }
- } catch (Exception e) {
- logger.error("error release lock:" + schedulerId());
- throw new RuntimeException(e);
- }
- }
-
- private String schedulerId() {
- return ZOOKEEPER_LOCK_PATH + "/" + jobEngineConfig.getConfig().getMetadataUrlPrefix();
- }
-
- private String getZKConnectString(JobEngineConfig context) {
- Configuration conf = HadoopUtil.newHBaseConfiguration(context.getConfig().getStorageUrl());
- return conf.get(HConstants.ZOOKEEPER_QUORUM) + ":" + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- }
-
public static DefaultScheduler getInstance() {
return INSTANCE;
}
@@ -178,33 +138,17 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
}
@Override
- public synchronized void init(JobEngineConfig jobEngineConfig) throws SchedulerException {
+ public synchronized void init(JobEngineConfig jobEngineConfig, final JobLock jobLock) throws SchedulerException {
if (!initialized) {
initialized = true;
} else {
return;
}
- String ZKConnectString = getZKConnectString(jobEngineConfig);
- if (StringUtils.isEmpty(ZKConnectString)) {
- throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
- }
this.jobEngineConfig = jobEngineConfig;
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
- this.zkClient.start();
- this.sharedLock = new InterProcessMutex(zkClient, schedulerId());
- boolean hasLock = false;
- try {
- hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
- } catch (Exception e) {
- logger.warn("error acquire lock", e);
- }
- if (!hasLock) {
- logger.warn("fail to acquire lock, scheduler has not been started");
- zkClient.close();
- return;
- }
+
+ jobLock.lock();
+
executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
//load all executable, set them to a consistent status
fetcherPool = Executors.newScheduledThreadPool(1);
@@ -224,6 +168,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
logger.debug("Closing zk connection");
try {
shutdown();
+ jobLock.unlock();
} catch (SchedulerException e) {
logger.error("error shutdown scheduler", e);
}
@@ -238,7 +183,6 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
public void shutdown() throws SchedulerException {
fetcherPool.shutdown();
jobPool.shutdown();
- releaseLock();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index d3c3669..74c824c 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -92,18 +93,17 @@ public class BuildCubeWithEngineTest {
DeployUtil.deployMetadata();
DeployUtil.overrideJobJarLocations();
-
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
jobService = ExecutableManager.getInstance(kylinConfig);
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
cubeManager = CubeManager.getInstance(kylinConfig);
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
+ if (jobService.getJob(jobId) instanceof CubingJob) {
jobService.deleteJob(jobId);
}
}
@@ -123,18 +123,12 @@ public class BuildCubeWithEngineTest {
}
private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
+ String[] testCase = new String[] { "testInnerJoinCube", "testInnerJoinCube2", };
runTestAndAssertSucceed(testCase);
}
private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
+ String[] testCase = new String[] { "testLeftJoinCube", "testLeftJoinCube2", };
runTestAndAssertSucceed(testCase);
}
@@ -187,7 +181,8 @@ public class BuildCubeWithEngineTest {
}
}
- @SuppressWarnings("unused") // called by reflection
+ @SuppressWarnings("unused")
+ // called by reflection
private List<String> testInnerJoinCube2() throws Exception {
clearSegment("test_kylin_cube_with_slr_empty");
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
@@ -201,11 +196,11 @@ public class BuildCubeWithEngineTest {
return result;
}
- @SuppressWarnings("unused") // called by reflection
+ @SuppressWarnings("unused")
+ // called by reflection
private List<String> testInnerJoinCube() throws Exception {
clearSegment("test_kylin_cube_without_slr_empty");
-
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
@@ -213,7 +208,6 @@ public class BuildCubeWithEngineTest {
long date1 = 0;
long date2 = f.parse("2013-01-01").getTime();
-
// this cube doesn't support incremental build, always do full build
List<String> result = Lists.newArrayList();
@@ -221,7 +215,8 @@ public class BuildCubeWithEngineTest {
return result;
}
- @SuppressWarnings("unused") // called by reflection
+ @SuppressWarnings("unused")
+ // called by reflection
private List<String> testLeftJoinCube2() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
@@ -243,7 +238,8 @@ public class BuildCubeWithEngineTest {
}
- @SuppressWarnings("unused") // called by reflection
+ @SuppressWarnings("unused")
+ // called by reflection
private List<String> testLeftJoinCube() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
@@ -263,10 +259,9 @@ public class BuildCubeWithEngineTest {
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
+ cubeManager.updateCube(cube, true);
}
-
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 612ba6c..1094caa 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -23,6 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -98,7 +99,7 @@ public class BuildIIWithEngineTest {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
jobService = ExecutableManager.getInstance(kylinConfig);
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig),new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
index f47e152..913f707 100644
--- a/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/impl/threadpool/BaseSchedulerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.job.impl.threadpool;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.ZookeeperJobLock;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -86,7 +87,7 @@ public abstract class BaseSchedulerTest extends HBaseMetadataTestCase {
setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"), 10);
jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index fca7e72..ed0901e 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.controller;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.lock.JobLock;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -53,6 +54,9 @@ public class JobController extends BasicController implements InitializingBean {
@Autowired
private JobService jobService;
+ @Autowired
+ private JobLock jobLock;
+
/*
* (non-Javadoc)
*
@@ -77,7 +81,7 @@ public class JobController extends BasicController implements InitializingBean {
public void run() {
try {
DefaultScheduler scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
+ scheduler.init(new JobEngineConfig(kylinConfig), jobLock);
while (!scheduler.hasStarted()) {
logger.error("scheduler has not been started");
Thread.sleep(1000);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
new file mode 100644
index 0000000..12cfc68
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -0,0 +1,25 @@
+package org.apache.kylin.rest.security;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+
+import java.io.IOException;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/19/15.
+ */
+public interface AclHBaseStorage {
+ String DEFAULT_TABLE_PREFIX = "kylin_metadata";
+
+ String ACL_INFO_FAMILY = "i";
+ String ACL_ACES_FAMILY = "a";
+ String ACL_TABLE_NAME = "_acl";
+
+ String USER_AUTHORITY_FAMILY = "a";
+ String USER_TABLE_NAME = "_user";
+ String USER_AUTHORITY_COLUMN = "c";
+
+ String prepareHBaseTable(Class clazz) throws IOException;
+
+ HTableInterface getTable(String tableName) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
new file mode 100644
index 0000000..f9c9e0c
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -0,0 +1,45 @@
+package org.apache.kylin.rest.security;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.kylin.common.util.MockHTable;
+import org.apache.kylin.rest.service.AclService;
+import org.apache.kylin.rest.service.UserService;
+import org.h2.util.StringUtils;
+
+import java.io.IOException;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/19/15.
+ */
+public class MockAclHBaseStorage implements AclHBaseStorage {
+
+ private HTableInterface mockedAclTable;
+ private HTableInterface mockedUserTable;
+ private static final String aclTableName = "MOCK-ACL-TABLE";
+ private static final String userTableName = "MOCK-USER-TABLE";
+
+ @Override
+ public String prepareHBaseTable(Class clazz) throws IOException {
+
+ if (clazz == AclService.class) {
+ mockedAclTable = new MockHTable(aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
+ return aclTableName;
+ } else if (clazz == UserService.class) {
+ mockedUserTable = new MockHTable(userTableName, USER_AUTHORITY_FAMILY);
+ return userTableName;
+ } else {
+ throw new IllegalStateException("prepareHBaseTable for unknown class: " + clazz);
+ }
+ }
+
+ @Override
+ public HTableInterface getTable(String tableName) throws IOException {
+ if (StringUtils.equals(tableName, aclTableName)) {
+ return mockedAclTable;
+ } else if (StringUtils.equals(tableName, userTableName)) {
+ return mockedUserTable;
+ } else {
+ throw new IllegalStateException("getTable failed" + tableName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
new file mode 100644
index 0000000..44e6f8f
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -0,0 +1,51 @@
+package org.apache.kylin.rest.security;
+
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
+import org.apache.kylin.rest.service.AclService;
+import org.apache.kylin.rest.service.UserService;
+import org.h2.util.StringUtils;
+
+import java.io.IOException;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 5/19/15.
+ */
+public class RealAclHBaseStorage implements AclHBaseStorage {
+
+ private String hbaseUrl;
+ private String aclTableName;
+ private String userTableName;
+
+ @Override
+ public String prepareHBaseTable(Class clazz) throws IOException {
+ String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
+ int cut = metadataUrl.indexOf('@');
+ String tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
+ hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+
+ if (clazz == AclService.class) {
+ aclTableName = tableNameBase + ACL_TABLE_NAME;
+ HBaseConnection.createHTableIfNeeded(hbaseUrl, aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
+ return aclTableName;
+ } else if (clazz == UserService.class) {
+ userTableName = tableNameBase + USER_TABLE_NAME;
+ HBaseConnection.createHTableIfNeeded(hbaseUrl, userTableName, USER_AUTHORITY_FAMILY);
+ return userTableName;
+ } else {
+ throw new IllegalStateException("prepareHBaseTable for unknown class: " + clazz);
+ }
+ }
+
+ @Override
+ public HTableInterface getTable(String tableName) throws IOException {
+ if (StringUtils.equals(tableName, aclTableName)) {
+ return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ } else if (StringUtils.equals(tableName, userTableName)) {
+ return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ } else {
+ throw new IllegalStateException("getTable failed" + tableName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/AclService.java b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
index 842233f..981205c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.rest.util.AclHBaseStorage;
+import org.apache.kylin.rest.security.AclHBaseStorage;
import org.apache.kylin.rest.util.Serializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/UserService.java b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
index bc838b6..cf6d03c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.client.*;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.security.UserManager;
-import org.apache.kylin.rest.util.AclHBaseStorage;
+import org.apache.kylin.rest.security.AclHBaseStorage;
import org.apache.kylin.rest.util.Serializer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.GrantedAuthority;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/util/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/AclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/util/AclHBaseStorage.java
deleted file mode 100644
index 7d495eb..0000000
--- a/server/src/main/java/org/apache/kylin/rest/util/AclHBaseStorage.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.apache.kylin.rest.util;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-
-import java.io.IOException;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 5/19/15.
- */
-public interface AclHBaseStorage {
- String DEFAULT_TABLE_PREFIX = "kylin_metadata";
-
- String ACL_INFO_FAMILY = "i";
- String ACL_ACES_FAMILY = "a";
- String ACL_TABLE_NAME = "_acl";
-
- String USER_AUTHORITY_FAMILY = "a";
- String USER_TABLE_NAME = "_user";
- String USER_AUTHORITY_COLUMN = "c";
-
- String prepareHBaseTable(Class clazz) throws IOException;
-
- HTableInterface getTable(String tableName) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/util/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/MockAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/util/MockAclHBaseStorage.java
deleted file mode 100644
index ce9ef1d..0000000
--- a/server/src/main/java/org/apache/kylin/rest/util/MockAclHBaseStorage.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.kylin.rest.util;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.kylin.common.util.MockHTable;
-import org.apache.kylin.rest.service.AclService;
-import org.apache.kylin.rest.service.UserService;
-import org.h2.util.StringUtils;
-
-import java.io.IOException;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 5/19/15.
- */
-public class MockAclHBaseStorage implements AclHBaseStorage {
-
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
- private static final String aclTableName = "MOCK-ACL-TABLE";
- private static final String userTableName = "MOCK-USER-TABLE";
-
- @Override
- public String prepareHBaseTable(Class clazz) throws IOException {
-
- if (clazz == AclService.class) {
- mockedAclTable = new MockHTable(aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
- return aclTableName;
- } else if (clazz == UserService.class) {
- mockedUserTable = new MockHTable(userTableName, USER_AUTHORITY_FAMILY);
- return userTableName;
- } else {
- throw new IllegalStateException("prepareHBaseTable for unknown class: " + clazz);
- }
- }
-
- @Override
- public HTableInterface getTable(String tableName) throws IOException {
- if (StringUtils.equals(tableName, aclTableName)) {
- return mockedAclTable;
- } else if (StringUtils.equals(tableName, userTableName)) {
- return mockedUserTable;
- } else {
- throw new IllegalStateException("getTable failed" + tableName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/java/org/apache/kylin/rest/util/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/util/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/util/RealAclHBaseStorage.java
deleted file mode 100644
index 4d4865e..0000000
--- a/server/src/main/java/org/apache/kylin/rest/util/RealAclHBaseStorage.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.kylin.rest.util;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.rest.service.AclService;
-import org.apache.kylin.rest.service.UserService;
-import org.h2.util.StringUtils;
-
-import java.io.IOException;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 5/19/15.
- */
-public class RealAclHBaseStorage implements AclHBaseStorage {
-
- private String hbaseUrl;
- private String aclTableName;
- private String userTableName;
-
- @Override
- public String prepareHBaseTable(Class clazz) throws IOException {
- String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
- int cut = metadataUrl.indexOf('@');
- String tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
- hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- if (clazz == AclService.class) {
- aclTableName = tableNameBase + ACL_TABLE_NAME;
- HBaseConnection.createHTableIfNeeded(hbaseUrl, aclTableName, ACL_INFO_FAMILY, ACL_ACES_FAMILY);
- return aclTableName;
- } else if (clazz == UserService.class) {
- userTableName = tableNameBase + USER_TABLE_NAME;
- HBaseConnection.createHTableIfNeeded(hbaseUrl, userTableName, USER_AUTHORITY_FAMILY);
- return userTableName;
- } else {
- throw new IllegalStateException("prepareHBaseTable for unknown class: " + clazz);
- }
- }
-
- @Override
- public HTableInterface getTable(String tableName) throws IOException {
- if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
- } else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
- } else {
- throw new IllegalStateException("getTable failed" + tableName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/main/resources/applicationContext.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml
index 619151d..aef311e 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -1,14 +1,13 @@
<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:context="http://www.springframework.org/schema/context"
- xmlns:mvc="http://www.springframework.org/schema/mvc"
- xmlns:task="http://www.springframework.org/schema/task"
- xmlns:metrics="http://www.ryantenney.com/schema/metrics"
- xmlns:aop="http://www.springframework.org/schema/aop"
- xmlns:cache="http://www.springframework.org/schema/cache"
- xmlns:p="http://www.springframework.org/schema/p"
- xmlns:jdbc="http://www.springframework.org/schema/jdbc"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:mvc="http://www.springframework.org/schema/mvc"
+ xmlns:task="http://www.springframework.org/schema/task"
+ xmlns:metrics="http://www.ryantenney.com/schema/metrics"
+ xmlns:aop="http://www.springframework.org/schema/aop"
+ xmlns:cache="http://www.springframework.org/schema/cache"
+ xmlns:p="http://www.springframework.org/schema/p"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.1.xsd
@@ -21,114 +20,119 @@
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
http://www.springframework.org/schema/cache
- http://www.springframework.org/schema/cache/spring-cache.xsd
- http://www.springframework.org/schema/jdbc
- http://www.springframework.org/schema/jdbc/spring-jdbc-3.0.xsd">
-
- <description>Kylin Rest Service</description>
- <context:annotation-config />
- <mvc:annotation-driven />
- <aop:aspectj-autoproxy />
-
- <context:component-scan base-package="org.apache.kylin.rest" />
-
- <!-- Matric Spring -->
- <!-- Registry should be defined in only one context XML file -->
- <metrics:metric-registry id="metrics" />
-
- <!-- annotation-driven must be included in all context files -->
- <metrics:annotation-driven metric-registry="metrics" />
-
- <!--<!– (Optional) Registry should be defined in only one context XML file –>-->
- <!--<metrics:reporter type="ganglia" metric-registry="metrics"-->
- <!--group="${ganglia.group}" port="${ganglia.port}" udp-mode="MULTICAST"-->
- <!--ttl="100" period="1m" />-->
-
- <!-- (Optional) The metrics in this example require the metrics-jvm jar -->
- <metrics:register metric-registry="metrics">
- <bean metrics:name="jvm.gc"
- class="com.codahale.metrics.jvm.GarbageCollectorMetricSet" />
- <bean metrics:name="jvm.memory" class="com.codahale.metrics.jvm.MemoryUsageGaugeSet" />
- <bean metrics:name="jvm.thread-states"
- class="com.codahale.metrics.jvm.ThreadStatesGaugeSet" />
- <!--<bean metrics:name="jvm.fd.usage" class="com.codahale.metrics.jvm.FileDescriptorRatioGauge"
- /> -->
- </metrics:register>
- <!-- Matric Spring -->
-
- <bean class="org.apache.kylin.rest.security.PasswordPlaceholderConfigurer">
- <property name="ignoreResourceNotFound" value="true" />
- </bean>
-
- <bean id="aclHBaseStorage" class="org.apache.kylin.rest.util.MockAclHBaseStorage"/>
-
- <!-- Rest service binding -->
- <bean
- class="org.springframework.web.servlet.mvc.annotation.DefaultAnnotationHandlerMapping" />
-
- <bean id="mappingJacksonHttpMessageConverter"
- class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter" />
- <bean id="stringHttpMessageConverter"
- class="org.springframework.http.converter.StringHttpMessageConverter" />
- <bean id="formHttpMessageConverter"
- class="org.springframework.http.converter.FormHttpMessageConverter" />
-
- <bean
- class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
- <property name="messageConverters">
- <list>
- <ref bean="mappingJacksonHttpMessageConverter" />
- <ref bean="stringHttpMessageConverter" />
- <ref bean="formHttpMessageConverter" />
- </list>
- </property>
- </bean>
-
- <bean
- class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
- <property name="mediaTypes">
- <map>
- <entry key="html" value="text/html" />
- <entry key="json" value="application/json" />
- </map>
- </property>
- <property name="viewResolvers">
- <list>
- <bean class="org.springframework.web.servlet.view.BeanNameViewResolver" />
- <bean
- class="org.springframework.web.servlet.view.InternalResourceViewResolver">
- <!-- <property name="prefix" value="/WEB-INF/jsp/"/> -->
- <property name="suffix" value=".jsp" />
- </bean>
- </list>
- </property>
- <property name="defaultViews">
- <list>
- <bean
- class="org.springframework.web.servlet.view.json.MappingJackson2JsonView" />
- </list>
- </property>
- </bean>
- <!-- Rest service binding -->
-
- <task:annotation-driven />
- <bean id="performanceMonitorTask" class="org.apache.kylin.rest.util.PerformanceMonitorTask"></bean>
-
- <!-- Cache Config -->
- <cache:annotation-driven />
- <bean id="cacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"
- p:cacheManager-ref="ehcache" />
- <!-- Cache Config -->
-
- <beans profile="default">
- <bean id="ehcache"
- class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
- p:configLocation="classpath:ehcache.xml" p:shared="true" />
- </beans>
-
- <beans profile="sandbox,testing">
- <bean id="ehcache"
- class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
- p:configLocation="classpath:ehcache-test.xml" p:shared="true" />
- </beans>
+ http://www.springframework.org/schema/cache/spring-cache.xsd">
+
+ <description>Kylin Rest Service</description>
+ <context:annotation-config/>
+ <mvc:annotation-driven/>
+ <aop:aspectj-autoproxy/>
+
+ <context:component-scan base-package="org.apache.kylin.rest"/>
+
+ <!-- Matric Spring -->
+ <!-- Registry should be defined in only one context XML file -->
+ <metrics:metric-registry id="metrics"/>
+
+ <!-- annotation-driven must be included in all context files -->
+ <metrics:annotation-driven metric-registry="metrics"/>
+
+ <!--<!– (Optional) Registry should be defined in only one context XML file –>-->
+ <!--<metrics:reporter type="ganglia" metric-registry="metrics"-->
+ <!--group="${ganglia.group}" port="${ganglia.port}" udp-mode="MULTICAST"-->
+ <!--ttl="100" period="1m" />-->
+
+ <!-- (Optional) The metrics in this example require the metrics-jvm jar -->
+ <metrics:register metric-registry="metrics">
+ <bean metrics:name="jvm.gc"
+ class="com.codahale.metrics.jvm.GarbageCollectorMetricSet"/>
+ <bean metrics:name="jvm.memory" class="com.codahale.metrics.jvm.MemoryUsageGaugeSet"/>
+ <bean metrics:name="jvm.thread-states"
+ class="com.codahale.metrics.jvm.ThreadStatesGaugeSet"/>
+ <!--<bean metrics:name="jvm.fd.usage" class="com.codahale.metrics.jvm.FileDescriptorRatioGauge"
+ /> -->
+ </metrics:register>
+ <!-- Matric Spring -->
+
+ <bean class="org.apache.kylin.rest.security.PasswordPlaceholderConfigurer">
+ <property name="ignoreResourceNotFound" value="true"/>
+ </bean>
+
+
+ <!-- Rest service binding -->
+ <bean
+ class="org.springframework.web.servlet.mvc.annotation.DefaultAnnotationHandlerMapping"/>
+
+ <bean id="mappingJacksonHttpMessageConverter"
+ class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"/>
+ <bean id="stringHttpMessageConverter"
+ class="org.springframework.http.converter.StringHttpMessageConverter"/>
+ <bean id="formHttpMessageConverter"
+ class="org.springframework.http.converter.FormHttpMessageConverter"/>
+
+ <bean
+ class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
+ <property name="messageConverters">
+ <list>
+ <ref bean="mappingJacksonHttpMessageConverter"/>
+ <ref bean="stringHttpMessageConverter"/>
+ <ref bean="formHttpMessageConverter"/>
+ </list>
+ </property>
+ </bean>
+
+ <bean
+ class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
+ <property name="mediaTypes">
+ <map>
+ <entry key="html" value="text/html"/>
+ <entry key="json" value="application/json"/>
+ </map>
+ </property>
+ <property name="viewResolvers">
+ <list>
+ <bean class="org.springframework.web.servlet.view.BeanNameViewResolver"/>
+ <bean
+ class="org.springframework.web.servlet.view.InternalResourceViewResolver">
+ <!-- <property name="prefix" value="/WEB-INF/jsp/"/> -->
+ <property name="suffix" value=".jsp"/>
+ </bean>
+ </list>
+ </property>
+ <property name="defaultViews">
+ <list>
+ <bean
+ class="org.springframework.web.servlet.view.json.MappingJackson2JsonView"/>
+ </list>
+ </property>
+ </bean>
+ <!-- Rest service binding -->
+ <task:annotation-driven/>
+ <bean id="performanceMonitorTask" class="org.apache.kylin.rest.util.PerformanceMonitorTask"></bean>
+
+ <!-- Cache Config -->
+ <cache:annotation-driven/>
+
+ <bean id="cacheManager" class="org.springframework.cache.ehcache.EhCacheCacheManager"
+ p:cacheManager-ref="ehcache"/>
+ <beans profile="default">
+ <bean id="ehcache"
+ class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
+ p:configLocation="classpath:ehcache.xml" p:shared="true"/>
+ </beans>
+ <beans profile="sandbox,testing">
+ <bean id="ehcache"
+ class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
+ p:configLocation="classpath:ehcache-test.xml" p:shared="true"/>
+ </beans>
+
+ <!-- hbase storage/global lock Config -->
+ <beans profile="default,sandbox">
+ <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.RealAclHBaseStorage"/>
+ <bean id="jobLock" class="org.apache.kylin.common.lock.ZookeeperJobLock"/>
+ </beans>
+ <beans profile="testing">
+ <bean id="aclHBaseStorage" class="org.apache.kylin.rest.security.MockAclHBaseStorage"/>
+ <bean id="jobLock" class="org.apache.kylin.common.lock.MockJobLock"/>
+ </beans>
+
</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5b6960ee/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 795afaf..ef834c0 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -64,7 +64,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
- createTestMetadata(LOCALMETA_TEST_DATA);
+ staticCreateTestMetadata(LOCALMETA_TEST_DATA);
configA = KylinConfig.getInstanceFromEnv();
configB = KylinConfig.getKylinConfigFromInputStream(KylinConfig.getKylinPropertiesAsInputSteam());
configB.setMetadataUrl("../examples/test_metadata");
[28/32] incubator-kylin git commit: KYLIN-697 refine zookeeperjoblock
Posted by ma...@apache.org.
KYLIN-697 refine zookeeperjoblock
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cf26d3d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cf26d3d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cf26d3d0
Branch: refs/heads/0.8.0
Commit: cf26d3d06a43d6ff87d6ecc0edc0a79285ab27cf
Parents: d647d5a
Author: honma <ho...@ebay.com>
Authored: Tue May 26 23:03:57 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:28 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/lock/ZookeeperJobLock.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf26d3d0/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
index 7a755a3..78da480 100644
--- a/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
+++ b/common/src/main/java/org/apache/kylin/common/lock/ZookeeperJobLock.java
@@ -23,11 +23,14 @@ public class ZookeeperJobLock implements JobLock {
private Logger logger = LoggerFactory.getLogger(ZookeeperJobLock.class);
private static final String ZOOKEEPER_LOCK_PATH = "/kylin/job_engine/lock";
+
+ private String scheduleID;
private InterProcessMutex sharedLock;
private CuratorFramework zkClient;
@Override
public boolean lock() {
+ this.scheduleID = schedulerId();
String ZKConnectString = getZKConnectString();
if (StringUtils.isEmpty(ZKConnectString)) {
throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
@@ -36,7 +39,7 @@ public class ZookeeperJobLock implements JobLock {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
this.zkClient = CuratorFrameworkFactory.newClient(ZKConnectString, retryPolicy);
this.zkClient.start();
- this.sharedLock = new InterProcessMutex(zkClient, schedulerId());
+ this.sharedLock = new InterProcessMutex(zkClient, this.scheduleID);
boolean hasLock = false;
try {
hasLock = sharedLock.acquire(3, TimeUnit.SECONDS);
@@ -65,12 +68,12 @@ public class ZookeeperJobLock implements JobLock {
try {
if (zkClient.getState().equals(CuratorFrameworkState.STARTED)) {
// client.setData().forPath(ZOOKEEPER_LOCK_PATH, null);
- if (zkClient.checkExists().forPath(schedulerId()) != null) {
- zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(schedulerId());
+ if (zkClient.checkExists().forPath(scheduleID) != null) {
+ zkClient.delete().guaranteed().deletingChildrenIfNeeded().forPath(scheduleID);
}
}
} catch (Exception e) {
- logger.error("error release lock:" + schedulerId());
+ logger.error("error release lock:" + scheduleID);
throw new RuntimeException(e);
}
}
[23/32] incubator-kylin git commit: KYLIN-697 backup all xmls for hdp
2.2.0
Posted by ma...@apache.org.
KYLIN-697 backup all xmls for hdp 2.2.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/82145364
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/82145364
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/82145364
Branch: refs/heads/0.8.0
Commit: 82145364da454147d4167fa14adba604c41338c1
Parents: 7d86446
Author: honma <ho...@ebay.com>
Authored: Tue May 26 11:03:13 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../test_case_data/sandbox-hdp220/.gitignore | 2 +
.../sandbox-hdp220/capacity-scheduler.xml | 99 +++
.../test_case_data/sandbox-hdp220/core-site.xml | 152 ++++
.../sandbox-hdp220/hadoop-policy.xml | 59 ++
.../sandbox-hdp220/hbase-policy.xml | 19 +
.../sandbox-hdp220/hbase-site.xml | 194 +++++
.../test_case_data/sandbox-hdp220/hdfs-site.xml | 265 +++++++
.../test_case_data/sandbox-hdp220/hive-site.xml | 794 +++++++++++++++++++
.../sandbox-hdp220/kylin.properties | 98 +++
.../sandbox-hdp220/kylin_hive_conf.xml | 54 ++
.../sandbox-hdp220/kylin_job_conf.xml | 61 ++
.../sandbox-hdp220/mapred-site.xml | 239 ++++++
.../test_case_data/sandbox-hdp220/yarn-site.xml | 509 ++++++++++++
13 files changed, 2545 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/.gitignore
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/.gitignore b/examples/test_case_data/sandbox-hdp220/.gitignore
new file mode 100644
index 0000000..5c501e0
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/.gitignore
@@ -0,0 +1,2 @@
+kylin.properties.override
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/capacity-scheduler.xml b/examples/test_case_data/sandbox-hdp220/capacity-scheduler.xml
new file mode 100644
index 0000000..7b4a367
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/capacity-scheduler.xml
@@ -0,0 +1,99 @@
+<!--Tue Dec 16 19:08:08 2014-->
+ <configuration>
+
+ <property>
+ <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.capacity</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+ <value> </value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-am-resource-percent</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/core-site.xml b/examples/test_case_data/sandbox-hdp220/core-site.xml
new file mode 100644
index 0000000..1b3a3ee
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/core-site.xml
@@ -0,0 +1,152 @@
+<!--Mon Jan 19 07:28:25 2015-->
+ <configuration>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>fs.trash.interval</name>
+ <value>360</value>
+ </property>
+
+ <property>
+ <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.falcon.groups</name>
+ <value>users</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.falcon.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hbase.groups</name>
+ <value>users</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hbase.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hcat.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hcat.hosts</name>
+ <value>sandbox.hortonworks.com</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hive.groups</name>
+ <value>users</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hive.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hue.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hue.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
+ <value>sandbox.hortonworks.com</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.root.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.root.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
+ RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
+ RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
+ RULE:[2:$1@$0](hm@.*)s/.*/hbase/
+ RULE:[2:$1@$0](rs@.*)s/.*/hbase/
+ DEFAULT
+ </value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ </property>
+
+ <property>
+ <name>io.file.buffer.size</name>
+ <value>131072</value>
+ </property>
+
+ <property>
+ <name>io.serializations</name>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ </property>
+
+ <property>
+ <name>ipc.client.connect.max.retries</name>
+ <value>50</value>
+ </property>
+
+ <property>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>ipc.client.idlethreshold</name>
+ <value>8000</value>
+ </property>
+
+ <property>
+ <name>ipc.server.tcpnodelay</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.webinterface.trusted</name>
+ <value>false</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/hadoop-policy.xml b/examples/test_case_data/sandbox-hdp220/hadoop-policy.xml
new file mode 100644
index 0000000..0f7e0b9
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/hadoop-policy.xml
@@ -0,0 +1,59 @@
+<!--Tue Dec 16 19:07:40 2014-->
+ <configuration>
+
+ <property>
+ <name>security.admin.operations.protocol.acl</name>
+ <value>hadoop</value>
+ </property>
+
+ <property>
+ <name>security.client.datanode.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.datanode.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.inter.datanode.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.inter.tracker.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.job.client.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.job.task.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.namenode.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.refresh.policy.protocol.acl</name>
+ <value>hadoop</value>
+ </property>
+
+ <property>
+ <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+ <value>hadoop</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/hbase-policy.xml b/examples/test_case_data/sandbox-hdp220/hbase-policy.xml
new file mode 100644
index 0000000..36756b8
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/hbase-policy.xml
@@ -0,0 +1,19 @@
+<!--Mon Jan 19 07:29:07 2015-->
+ <configuration>
+
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/hbase-site.xml b/examples/test_case_data/sandbox-hdp220/hbase-site.xml
new file mode 100644
index 0000000..9fd97c9
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/hbase-site.xml
@@ -0,0 +1,194 @@
+<!--Mon Jan 19 07:29:07 2015-->
+ <configuration>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ </property>
+
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
+ </property>
+
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ </property>
+
+ <property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ </property>
+
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value>604800000</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.majorcompaction.jitter</name>
+ <value>0.50</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>10737418240</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value>4</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value>134217728</value>
+ </property>
+
+ <property>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local</value>
+ </property>
+
+ <property>
+ <name>hbase.master.info.bindAddress</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>60010</value>
+ </property>
+
+ <property>
+ <name>hbase.master.port</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <value>0.4</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>60</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>60030</value>
+ </property>
+
+ <property>
+ <name>hbase.rootdir</name>
+ <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ </property>
+
+ <property>
+ <name>hbase.rpc.engine</name>
+ <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
+ </property>
+
+ <property>
+ <name>hbase.rpc.protection</name>
+ <value>PRIVACY</value>
+ </property>
+
+ <property>
+ <name>hbase.security.authentication</name>
+ <value>simple</value>
+ </property>
+
+ <property>
+ <name>hbase.security.authorization</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.superuser</name>
+ <value>hbase</value>
+ </property>
+
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>/hadoop/hbase</value>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>sandbox.hortonworks.com</value>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.useMulti</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase_master_heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>hbase_regionserver_heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>hfile.block.cache.size</name>
+ <value>0.40</value>
+ </property>
+
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase-unsecure</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/hdfs-site.xml b/examples/test_case_data/sandbox-hdp220/hdfs-site.xml
new file mode 100644
index 0000000..f8347dc
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/hdfs-site.xml
@@ -0,0 +1,265 @@
+<!--Mon Jan 19 07:29:07 2015-->
+ <configuration>
+
+ <property>
+ <name>dfs.block.access.token.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.block.size</name>
+ <value>34217472</value>
+ </property>
+
+ <property>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>120</value>
+ </property>
+
+ <property>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
+ </property>
+
+ <property>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+ <value>4096</value>
+ </property>
+
+ <property>
+ <name>dfs.cluster.administrators</name>
+ <value> hdfs</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:50010</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.balance.bandwidthPerSec</name>
+ <value>6250000</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/hadoop/hdfs/data</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>750</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.du.reserved</name>
+ <value>1073741824</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.failed.volumes.tolerated</name>
+ <value>0</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:50075</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.https.address</name>
+ <value>0.0.0.0:50475</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:8010</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ </property>
+
+ <property>
+ <name>dfs.heartbeat.interval</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>dfs.hosts.exclude</name>
+ <value>/etc/hadoop/conf/dfs.exclude</value>
+ </property>
+
+ <property>
+ <name>dfs.http.policy</name>
+ <value>HTTP_ONLY</value>
+ </property>
+
+ <property>
+ <name>dfs.https.port</name>
+ <value>50470</value>
+ </property>
+
+ <property>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>/hadoop/hdfs/journalnode</value>
+ </property>
+
+ <property>
+ <name>dfs.journalnode.http-address</name>
+ <value>0.0.0.0:8480</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.accesstime.precision</name>
+ <value>3600000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.avoid.read.stale.datanode</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.avoid.write.stale.datanode</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.checkpoint.dir</name>
+ <value>/hadoop/hdfs/namesecondary</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.checkpoint.edits.dir</name>
+ <value>${dfs.namenode.checkpoint.dir}</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.checkpoint.period</name>
+ <value>21600</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.checkpoint.txns</name>
+ <value>1000000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.handler.count</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.http-address</name>
+ <value>sandbox.hortonworks.com:50070</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.namenode.https-address</name>
+ <value>sandbox.hortonworks.com:50470</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/hadoop/hdfs/namenode</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.namenode.name.dir.restore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.safemode.threshold-pct</name>
+ <value>1.0f</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.secondary.http-address</name>
+ <value>sandbox.hortonworks.com:50090</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.stale.datanode.interval</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.startup.delay.block.deletion.sec</name>
+ <value>3600</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.write.stale.datanode.ratio</name>
+ <value>1.0f</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs3.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+ <property>
+ <name>dfs.permissions.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.permissions.superusergroup</name>
+ <value>hdfs</value>
+ </property>
+
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>dfs.replication.max</name>
+ <value>50</value>
+ </property>
+
+ <property>
+ <name>dfs.support.append</name>
+ <value>true</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>true</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/hive-site.xml b/examples/test_case_data/sandbox-hdp220/hive-site.xml
new file mode 100644
index 0000000..40afbf6
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/hive-site.xml
@@ -0,0 +1,794 @@
+<!--Tue Dec 16 19:33:41 2014-->
+ <configuration>
+
+ <property>
+ <name>ambari.hive.db.schema.name</name>
+ <value>hive</value>
+ </property>
+
+ <property>
+ <name>datanucleus.cache.level2.type</name>
+ <value>none</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>1000000000</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.sortmerge.join</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.sortmerge.join.noconditionaltask</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.cbo.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.cli.print.header</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.cluster.delegation.token.store.class</name>
+ <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value>
+ </property>
+
+ <property>
+ <name>hive.cluster.delegation.token.store.zookeeper.connectString</name>
+ <value>sandbox.hortonworks.com:2181</value>
+ </property>
+
+ <property>
+ <name>hive.cluster.delegation.token.store.zookeeper.znode</name>
+ <value>/hive/cluster/delegation</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.abortedtxn.threshold</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.check.interval</name>
+ <value>300s</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.delta.num.threshold</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.delta.pct.threshold</name>
+ <value>0.1f</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.initiator.on</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.worker.threads</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.worker.timeout</name>
+ <value>86400s</value>
+ </property>
+
+ <property>
+ <name>hive.compute.query.using.stats</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.conf.restricted.list</name>
+ <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value>
+ </property>
+
+ <property>
+ <name>hive.convert.join.bucket.mapjoin.tez</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.enforce.bucketing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.enforce.sorting</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.enforce.sortmergebucketmapjoin</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.exec.compress.intermediate</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.exec.compress.output</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.exec.dynamic.partition</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.exec.dynamic.partition.mode</name>
+ <value>nonstrict</value>
+ </property>
+
+ <property>
+ <name>hive.exec.failure.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.max.created.files</name>
+ <value>100000</value>
+ </property>
+
+ <property>
+ <name>hive.exec.max.dynamic.partitions</name>
+ <value>5000</value>
+ </property>
+
+ <property>
+ <name>hive.exec.max.dynamic.partitions.pernode</name>
+ <value>2000</value>
+ </property>
+
+ <property>
+ <name>hive.exec.orc.compression.strategy</name>
+ <value>SPEED</value>
+ </property>
+
+ <property>
+ <name>hive.exec.orc.default.compress</name>
+ <value>ZLIB</value>
+ </property>
+
+ <property>
+ <name>hive.exec.orc.default.stripe.size</name>
+ <value>67108864</value>
+ </property>
+
+ <property>
+ <name>hive.exec.parallel</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.exec.parallel.thread.number</name>
+ <value>8</value>
+ </property>
+
+ <property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.reducers.bytes.per.reducer</name>
+ <value>67108864</value>
+ </property>
+
+ <property>
+ <name>hive.exec.reducers.max</name>
+ <value>1009</value>
+ </property>
+
+ <property>
+ <name>hive.exec.scratchdir</name>
+ <value>/tmp/hive</value>
+ </property>
+
+ <property>
+ <name>hive.exec.submit.local.task.via.child</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.exec.submitviachild</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.execution.engine</name>
+ <value>mr</value>
+ </property>
+
+ <property>
+ <name>hive.fetch.task.aggr</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.fetch.task.conversion</name>
+ <value>more</value>
+ </property>
+
+ <property>
+ <name>hive.fetch.task.conversion.threshold</name>
+ <value>1073741824</value>
+ </property>
+
+ <property>
+ <name>hive.heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>hive.limit.optimize.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.limit.pushdown.memory.usage</name>
+ <value>0.04</value>
+ </property>
+
+ <property>
+ <name>hive.map.aggr</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.map.aggr.hash.force.flush.memory.threshold</name>
+ <value>0.9</value>
+ </property>
+
+ <property>
+ <name>hive.map.aggr.hash.min.reduction</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
+ <name>hive.map.aggr.hash.percentmemory</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
+ <name>hive.mapjoin.bucket.cache.size</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>hive.mapjoin.optimized.hashtable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.mapred.reduce.tasks.speculative.execution</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.merge.mapfiles</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.merge.mapredfiles</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.merge.orcfile.stripe.level</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.merge.rcfile.block.level</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.merge.size.per.task</name>
+ <value>256000000</value>
+ </property>
+
+ <property>
+ <name>hive.merge.smallfiles.avgsize</name>
+ <value>16000000</value>
+ </property>
+
+ <property>
+ <name>hive.merge.tezfiles</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.authorization.storage.checks</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.cache.pinobjtypes</name>
+ <value>Table,Database,Type,FieldSchema,Order</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.client.connect.retry.delay</name>
+ <value>5s</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.client.socket.timeout</name>
+ <value>1800s</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.connect.retries</name>
+ <value>24</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.execute.setugi</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.failure.retries</name>
+ <value>24</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.kerberos.keytab.file</name>
+ <value>/etc/security/keytabs/hive.service.keytab</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.kerberos.principal</name>
+ <value>hive/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.pre.event.listeners</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.sasl.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.server.max.threads</name>
+ <value>100000</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://sandbox.hortonworks.com:9083</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>/apps/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.bucketmapjoin</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.bucketmapjoin.sortedmerge</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.constant.propagation</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.index.filter</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.mapjoin.mapreduce</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.metadataonly</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.null.scan</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.reducededuplication</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.reducededuplication.min.reducer</name>
+ <value>4</value>
+ </property>
+
+ <property>
+ <name>hive.optimize.sort.dynamic.partition</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.orc.compute.splits.num.threads</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>hive.orc.splits.include.file.footer</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.prewarm.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.prewarm.numcontainers</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>hive.security.authenticator.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value>
+ </property>
+
+ <property>
+ <name>hive.security.authorization.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.security.authorization.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value>
+ </property>
+
+ <property>
+ <name>hive.security.metastore.authenticator.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value>
+ </property>
+
+ <property>
+ <name>hive.security.metastore.authorization.auth.reads</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.security.metastore.authorization.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
+ </property>
+
+ <property>
+ <name>hive.server2.allow.user.substitution</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.authentication</name>
+ <value>NONE</value>
+ </property>
+
+ <property>
+ <name>hive.server2.authentication.spnego.keytab</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ </property>
+
+ <property>
+ <name>hive.server2.authentication.spnego.principal</name>
+ <value>/etc/security/keytabs/spnego.service.keytab</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.doAs</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.impersonation</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.logging.operation.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.logging.operation.log.location</name>
+ <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
+ </property>
+
+ <property>
+ <name>hive.server2.support.dynamic.service.discovery</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.table.type.mapping</name>
+ <value>CLASSIC</value>
+ </property>
+
+ <property>
+ <name>hive.server2.tez.default.queues</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <name>hive.server2.tez.initialize.default.sessions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.http.path</name>
+ <value>cliservice</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.http.port</name>
+ <value>10001</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.max.worker.threads</name>
+ <value>500</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.sasl.qop</name>
+ <value>auth</value>
+ </property>
+
+ <property>
+ <name>hive.server2.transport.mode</name>
+ <value>binary</value>
+ </property>
+
+ <property>
+ <name>hive.server2.use.SSL</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.server2.zookeeper.namespace</name>
+ <value>hiveserver2</value>
+ </property>
+
+ <property>
+ <name>hive.smbjoin.cache.rows</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>hive.stats.autogather</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.stats.dbclass</name>
+ <value>fs</value>
+ </property>
+
+ <property>
+ <name>hive.stats.fetch.column.stats</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.stats.fetch.partition.stats</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.tez.auto.reducer.parallelism</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.tez.container.size</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>hive.tez.cpu.vcores</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>hive.tez.dynamic.partition.pruning</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.tez.dynamic.partition.pruning.max.data.size</name>
+ <value>104857600</value>
+ </property>
+
+ <property>
+ <name>hive.tez.dynamic.partition.pruning.max.event.size</name>
+ <value>1048576</value>
+ </property>
+
+ <property>
+ <name>hive.tez.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value>
+ </property>
+
+ <property>
+ <name>hive.tez.java.opts</name>
+ <value>-server -Xmx200m -Djava.net.preferIPv4Stack=true</value>
+ </property>
+
+ <property>
+ <name>hive.tez.log.level</name>
+ <value>INFO</value>
+ </property>
+
+ <property>
+ <name>hive.tez.max.partition.factor</name>
+ <value>2.0</value>
+ </property>
+
+ <property>
+ <name>hive.tez.min.partition.factor</name>
+ <value>0.25</value>
+ </property>
+
+ <property>
+ <name>hive.tez.smb.number.waves</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
+ <name>hive.txn.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
+ </property>
+
+ <property>
+ <name>hive.txn.max.open.batch</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>hive.txn.timeout</name>
+ <value>300</value>
+ </property>
+
+ <property>
+ <name>hive.user.install.directory</name>
+ <value>/user/</value>
+ </property>
+
+ <property>
+ <name>hive.users.in.admin.role</name>
+ <value>hue,hive</value>
+ </property>
+
+ <property>
+ <name>hive.vectorized.execution.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.vectorized.execution.reduce.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.vectorized.groupby.checkinterval</name>
+ <value>4096</value>
+ </property>
+
+ <property>
+ <name>hive.vectorized.groupby.flush.percent</name>
+ <value>0.1</value>
+ </property>
+
+ <property>
+ <name>hive.vectorized.groupby.maxentries</name>
+ <value>100000</value>
+ </property>
+
+ <property>
+ <name>hive.zookeeper.client.port</name>
+ <value>2181</value>
+ </property>
+
+ <property>
+ <name>hive.zookeeper.namespace</name>
+ <value>hive_zookeeper_namespace</value>
+ </property>
+
+ <property>
+ <name>hive.zookeeper.quorum</name>
+ <value>sandbox.hortonworks.com:2181</value>
+ </property>
+
+ <property>
+ <name>hive_metastore_user_passwd</name>
+ <value>hive</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>hive</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>hive</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/kylin.properties b/examples/test_case_data/sandbox-hdp220/kylin.properties
new file mode 100644
index 0000000..f211137
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/kylin.properties
@@ -0,0 +1,98 @@
+## Config for Kylin Engine ##
+
+# List of web servers in use, this enables one web server instance to sync up with other servers.
+kylin.rest.servers=localhost:7070
+
+# The metadata store in hbase
+kylin.metadata.url=kylin_metadata@hbase
+
+# The storage for final cube file in hbase
+kylin.storage.url=hbase
+
+# Temp folder in hdfs
+kylin.hdfs.working.dir=/tmp
+
+kylin.job.mapreduce.default.reduce.input.mb=500
+
+# If true, job engine will not assume that hadoop CLI reside on the same server as it self
+# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
+kylin.job.run.as.remote.cmd=false
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.hostname=
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.username=
+
+# Only necessary when kylin.job.run.as.remote.cmd=true
+kylin.job.remote.cli.password=
+
+# Used by test cases to prepare synthetic data for sample cube
+kylin.job.remote.cli.working.dir=/tmp/kylin
+
+# Max count of concurrent jobs running
+kylin.job.concurrent.max.limit=10
+
+# Whether calculate cube in mem in each mapper;
+kylin.job.cubing.inMem=true
+
+#the percentage of the sampling, default 25%
+kylin.job.cubing.inMem.sampling.percent=25
+
+# The cut size for hbase region, in GB.
+# E.g, for cube whose capacity be marked as "SMALL", split region per 5GB by default
+kylin.job.hbase.region.cut.small=5
+kylin.job.hbase.region.cut.medium=10
+kylin.job.hbase.region.cut.large=50
+
+# Time interval to check hadoop job status
+kylin.job.yarn.app.rest.check.interval.seconds=10
+
+## Config for Restful APP ##
+# database connection settings:
+ldap.server=
+ldap.username=
+ldap.password=
+ldap.user.searchBase=
+ldap.user.searchPattern=
+ldap.user.groupSearchBase=
+ldap.service.searchBase=OU=
+ldap.service.searchPattern=
+ldap.service.groupSearchBase=
+acl.adminRole=
+acl.defaultRole=
+ganglia.group=
+ganglia.port=8664
+
+## Config for mail service
+
+# If true, will send email notification;
+mail.enabled=false
+mail.host=
+mail.username=
+mail.password=
+mail.sender=
+
+###########################config info for web#######################
+
+#help info ,format{name|displayName|link} ,optional
+kylin.web.help.length=4
+kylin.web.help.0=start|Getting Started|
+kylin.web.help.1=odbc|ODBC Driver|
+kylin.web.help.2=tableau|Tableau Guide|
+kylin.web.help.3=onboard|Cube Design Tutorial|
+#hadoop url link ,optional
+kylin.web.hadoop=
+#job diagnostic url link ,optional
+kylin.web.diagnostic=
+#contact mail on web page ,optional
+kylin.web.contact_mail=
+
+###########################config info for front#######################
+
+#env DEV|QA|PROD
+deploy.env=DEV
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/kylin_hive_conf.xml b/examples/test_case_data/sandbox-hdp220/kylin_hive_conf.xml
new file mode 100644
index 0000000..d990175
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/kylin_hive_conf.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>2</value>
+ <description>Block replication</description>
+ </property>
+
+ <property>
+ <name>hive.exec.compress.output</name>
+ <value>true</value>
+ <description>enable compress</description>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask</name>
+ <value>true</value>
+ <description>enable map-side join</description>
+ </property>
+
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>300000000</value>
+ <description>enable map-side join</description>
+ </property>
+
+ <!--
+ <property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description></description>
+ </property>
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description></description>
+ </property>
+ -->
+ <property>
+ <name>hive.merge.mapfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on mapper only job</description>
+ </property>
+ <property>
+ <name>hive.merge.mapredfiles</name>
+ <value>true</value>
+ <description>Enable hive file merge on map-reduce job</description>
+ </property>
+ <property>
+ <name>hive.merge.size.per.task</name>
+ <value>64000000</value>
+ <description>Size for the merged file</description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/kylin_job_conf.xml b/examples/test_case_data/sandbox-hdp220/kylin_job_conf.xml
new file mode 100644
index 0000000..f811ed4
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/kylin_job_conf.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<configuration>
+
+ <property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>-1</value>
+ <description>The maximum permissible size of the split metainfo file.
+ The JobTracker won't attempt to read split metainfo files bigger than
+ the configured value. No limits if set to -1.
+ </description>
+ </property>
+
+ <!-- uncomment the following 5 properties to enable lzo compressing
+
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
+
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
+
+ <property>
+ <name>mapred.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ !-->
+
+ <property>
+ <name>mapreduce.job.max.split.locations</name>
+ <value>2000</value>
+ <description>No description</description>
+ </property>
+
+ <property>
+ <name>dfs.replication</name>
+ <value>2</value>
+ <description>Block replication</description>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/mapred-site.xml b/examples/test_case_data/sandbox-hdp220/mapred-site.xml
new file mode 100644
index 0000000..ee3837c
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/mapred-site.xml
@@ -0,0 +1,239 @@
+<!--Tue Dec 16 19:08:08 2014-->
+ <configuration>
+
+ <property>
+ <name>io.sort.mb</name>
+ <value>64</value>
+ </property>
+
+ <property>
+ <name>mapred.child.java.opts</name>
+ <value>-Xmx200m</value>
+ </property>
+
+ <property>
+ <name>mapred.job.map.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>mapred.job.reduce.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ </property>
+
+ <property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+ </property>
+
+ <property>
+ <name>mapreduce.application.classpath</name>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*,$PWD/mr-framework/hadoop/share/hadoop/common/*,$PWD/mr-framework/hadoop/share/hadoop/common/lib/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar,/etc/hadoop/conf/secure</value>
+ </property>
+
+ <property>
+ <name>mapreduce.application.framework.path</name>
+ <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
+ </property>
+
+ <property>
+ <name>mapreduce.cluster.administrators</name>
+ <value> hadoop</value>
+ </property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>mapreduce.job.emit-timeline-data</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.05</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>sandbox.hortonworks.com:10020</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>sandbox.hortonworks.com:19888</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Xmx200m</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.log.level</name>
+ <value>INFO</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.output.compress</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.7</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.input.buffer.percent</name>
+ <value>0.0</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Xmx200m</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.log.level</name>
+ <value>INFO</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
+ <value>0.7</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>30</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>mapreduce.shuffle.port</name>
+ <value>13562</value>
+ </property>
+
+ <property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>64</value>
+ </property>
+
+ <property>
+ <name>mapreduce.task.timeout</name>
+ <value>300000</value>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-Dhdp.version=${hdp.version}</value>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Xmx200m</value>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.log.level</name>
+ <value>INFO</value>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+ </property>
+
+ </configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/82145364/examples/test_case_data/sandbox-hdp220/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp220/yarn-site.xml b/examples/test_case_data/sandbox-hdp220/yarn-site.xml
new file mode 100644
index 0000000..4e0ed9c
--- /dev/null
+++ b/examples/test_case_data/sandbox-hdp220/yarn-site.xml
@@ -0,0 +1,509 @@
+<!--Tue Dec 16 19:08:08 2014-->
+ <configuration>
+
+ <property>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>sandbox.hortonworks.com:2181</value>
+ </property>
+
+ <property>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.admin.acl</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
+ </property>
+
+ <property>
+ <name>yarn.client.nodemanager-connect.max-wait-ms</name>
+ <value>900000</value>
+ </property>
+
+ <property>
+ <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
+ </property>
+
+ <property>
+ <name>yarn.log.server.url</name>
+ <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
+ </property>
+
+ <property>
+ <name>yarn.node-labels.fs-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ </property>
+
+ <property>
+ <name>yarn.node-labels.fs-store.root-dir</name>
+ <value>/system/yarn/node-labels</value>
+ </property>
+
+ <property>
+ <name>yarn.node-labels.manager-class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.address</name>
+ <value>0.0.0.0:45454</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.admin-env</name>
+ <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.container-monitor.interval-ms</name>
+ <value>3000</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>0</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>90</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
+ <value>0.25</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.health-checker.interval-ms</name>
+ <value>135000</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+ <value>hadoop-yarn</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/hadoop/yarn/local</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/hadoop/yarn/log</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log.retain-second</name>
+ <value>604800</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.recovery.dir</name>
+ <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.recovery.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/app-logs</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>8</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>2250</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox.hortonworks.com:8050</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>sandbox.hortonworks.com:8141</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.connect.max-wait.ms</name>
+ <value>900000</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
+ <value> </value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.ha.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>sandbox.hortonworks.com</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.nodes.exclude-path</name>
+ <value>/etc/hadoop/conf/yarn.exclude</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.recovery.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>sandbox.hortonworks.com:8025</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>sandbox.hortonworks.com:8030</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.state-store.max-completed-applications</name>
+ <value>${yarn.resourcemanager.max-completed-applications}</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.store.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>sandbox.hortonworks.com:8088</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-acl</name>
+ <value>world:anyone:rwcda </value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>localhost:2181</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-num-retries</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-state-store.parent-path</name>
+ <value>/rmstore</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.zk-timeout-ms</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2250</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.address</name>
+ <value>sandbox.hortonworks.com:10200</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.bind-host</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.client.max-retries</name>
+ <value>30</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.client.retry-interval-ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.generic-application-history.store-class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
+ <value>104857600</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <value>10000</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
+ <value>300000</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.ttl-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.ttl-ms</name>
+ <value>2678400000</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.webapp.address</name>
+ <value>sandbox.hortonworks.com:8188</value>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ <value>sandbox.hortonworks.com:8190</value>
+ </property>
+
+ </configuration>
[32/32] incubator-kylin git commit: KYLIN-698 ignore
InMemCubeBuilderBenchmarkTest
Posted by ma...@apache.org.
KYLIN-698 ignore InMemCubeBuilderBenchmarkTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/74572f3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/74572f3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/74572f3e
Branch: refs/heads/0.8.0
Commit: 74572f3eee3ccb840830e0d6b9df8a595d227efb
Parents: cf26d3d
Author: honma <ho...@ebay.com>
Authored: Tue May 26 23:25:24 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:25:24 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java | 2 ++
pom.xml | 1 +
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/74572f3e/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java
index e90a41b..f61aa66 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilderBenchmarkTest.java
@@ -13,6 +13,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.gridtable.GTRecord;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +33,7 @@ import java.util.concurrent.LinkedBlockingQueue;
/**
*/
+@Ignore
public class InMemCubeBuilderBenchmarkTest extends LocalFileMetadataTestCase {
private static final Logger logger = LoggerFactory.getLogger(InMemCubeBuilderBenchmarkTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/74572f3e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index adfda4c..3f24301 100644
--- a/pom.xml
+++ b/pom.xml
@@ -645,6 +645,7 @@
<excludes>
<exclude>**/*$*</exclude>
<exclude>**/ITKafka*.java</exclude>
+
</excludes>
<systemProperties>
<property>
[14/32] incubator-kylin git commit: KYLIN-697 exclude inner classes
in failover
Posted by ma...@apache.org.
KYLIN-697 exclude inner classes in failover
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ca6a99a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ca6a99a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ca6a99a5
Branch: refs/heads/0.8.0
Commit: ca6a99a5ea56a4cd8ec44ecb2138c258cf6cfdee
Parents: 0d859dd
Author: honma <ho...@ebay.com>
Authored: Tue May 26 10:34:28 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6a99a5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 81f1d97..204a5a4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -639,6 +639,7 @@
</executions>
<configuration>
<excludes>
+ <exclude>**/*$*</exclude>
<exclude>**/ITKafka*.java</exclude>
</excludes>
<systemProperties>
[18/32] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3a0d9d20/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
new file mode 100644
index 0000000..cfb7bdb
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class ITBuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
[15/32] incubator-kylin git commit: fix ci: try travis ci
Posted by ma...@apache.org.
fix ci: try travis ci
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0d859dda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0d859dda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0d859dda
Branch: refs/heads/0.8.0
Commit: 0d859dda2b52e7fec0abd9c4b96934c9a1f00ec6
Parents: 29058ab
Author: honma <ho...@ebay.com>
Authored: Mon May 25 16:27:50 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.travis.yml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0d859dda/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..f5c99a7
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1 @@
+language: java
\ No newline at end of file
[20/32] incubator-kylin git commit: KYLIN-697 ignore problematic test
cases
Posted by ma...@apache.org.
KYLIN-697 ignore problematic test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2964a034
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2964a034
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2964a034
Branch: refs/heads/0.8.0
Commit: 2964a03482e9704f8713eedd88148068607f08b4
Parents: ee8959c
Author: honma <ho...@ebay.com>
Authored: Tue May 26 13:22:38 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java | 1 +
.../org/apache/kylin/rest/controller/ITTableControllerTest.java | 3 +++
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2964a034/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
index 756a38c..d7aa856 100644
--- a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -50,6 +50,7 @@ import java.io.IOException;
/**
*/
+@Ignore("this test case will break existing metadata store")
public class ITKafkaBasedIIStreamBuilderTest {
private static final Logger logger = LoggerFactory.getLogger(ITKafkaBasedIIStreamBuilderTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2964a034/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
index 75defbf..7e6dd5a 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
@@ -23,6 +23,7 @@ import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -33,6 +34,8 @@ import java.util.Map;
/**
* @author shaoshi
*/
+//TODO: break this into unit tests
+@Ignore("ServiceTestBase has been changed to local meta tests, thus not suitable for hive related tests")
public class ITTableControllerTest extends ServiceTestBase {
private TableController tableController;
[27/32] incubator-kylin git commit: KYLIN-783 minor change to trigger
mirroring
Posted by ma...@apache.org.
KYLIN-783 minor change to trigger mirroring
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0c5d549b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0c5d549b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0c5d549b
Branch: refs/heads/0.8.0
Commit: 0c5d549ba55a4d13b74fd83dd72656b3b4eb9dfa
Parents: 69c1a93
Author: honma <ho...@ebay.com>
Authored: Tue May 26 15:46:38 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:28 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/ITTableControllerTest.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0c5d549b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
index 7e6dd5a..6fd7653 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
@@ -65,10 +65,8 @@ public class ITTableControllerTest extends ServiceTestBase {
break;
}
}
-
Assert.assertNotNull(factTable);
-
Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
Assert.assertNotNull(loadResult);
[25/32] incubator-kylin git commit: KYLIN-783 append hive path to
mapred-site.xml
Posted by ma...@apache.org.
KYLIN-783 append hive path to mapred-site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/84e4d7ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/84e4d7ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/84e4d7ff
Branch: refs/heads/0.8.0
Commit: 84e4d7ffa176e3c090577f6a1596bb4d762e091f
Parents: 8214536
Author: honma <ho...@ebay.com>
Authored: Tue May 26 11:15:53 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/84e4d7ff/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index f6a1c2e..674643a 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -23,7 +23,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
[21/32] incubator-kylin git commit: KYLIN-793 modify all xmls
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 4e0ed9c..f764f1b 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -1,509 +1,484 @@
-<!--Tue Dec 16 19:08:08 2014-->
- <configuration>
-
+<!--Mon May 25 01:30:55 2015-->
+<configuration>
+
<property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>sandbox.hortonworks.com:2181</value>
</property>
-
+
<property>
- <name>yarn.acl.enable</name>
- <value>false</value>
+ <name>yarn.acl.enable</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.admin.acl</name>
- <value></value>
+ <name>yarn.admin.acl</name>
+ <value></value>
</property>
-
+
<property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
</property>
-
+
<property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>900000</value>
+ <name>yarn.client.nodemanager-connect.max-wait-ms</name>
+ <value>900000</value>
</property>
-
+
<property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
+ <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>2592000</value>
</property>
-
+
<property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
+ <name>yarn.log.server.url</name>
+ <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
</property>
-
+
<property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
+ <name>yarn.node-labels.fs-store.retry-policy-spec</name>
+ <value>2000, 500</value>
</property>
-
+
<property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
+ <name>yarn.node-labels.fs-store.root-dir</name>
+ <value>/system/yarn/node-labels</value>
</property>
-
+
<property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
+ <name>yarn.node-labels.manager-class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
+ <name>yarn.nodemanager.address</name>
+ <value>0.0.0.0:45454</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
+ <name>yarn.nodemanager.admin-env</name>
+ <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.nodemanager.bind-host</name>
+ <value>0.0.0.0</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
+ <name>yarn.nodemanager.container-monitor.interval-ms</name>
+ <value>3000</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>0</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>90</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
+ <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
+ <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
+ <value>0.25</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
+ <name>yarn.nodemanager.health-checker.interval-ms</name>
+ <value>135000</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
+ <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
+ <value>60000</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+ <value>hadoop-yarn</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
+ <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/hadoop/yarn/local</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
+ <name>yarn.nodemanager.log-aggregation.compression-type</name>
+ <value>gz</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
+ <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
+ <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <value>-1</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/hadoop/yarn/log</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
+ <name>yarn.nodemanager.log.retain-second</name>
+ <value>604800</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
+ <name>yarn.nodemanager.recovery.dir</name>
+ <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
+ <name>yarn.nodemanager.recovery.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.recovery.enabled</name>
- <value>false</value>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/app-logs</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
+ <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
+ <value>logs</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>8</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>14336</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>2250</value>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
+ <name>yarn.nodemanager.vmem-pmem-ratio</name>
+ <value>2.1</value>
</property>
-
+
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
+ <name>yarn.resourcemanager.address</name>
+ <value>sandbox.hortonworks.com:8050</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
+ <name>yarn.resourcemanager.admin.address</name>
+ <value>sandbox.hortonworks.com:8141</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
+ <name>yarn.resourcemanager.bind-host</name>
+ <value>0.0.0.0</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.resourcemanager.connect.max-wait.ms</name>
+ <value>900000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
+ <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+ <value>30000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
+ <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
+ <value>2000, 500</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
+ <value> </value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value> </value>
+ <name>yarn.resourcemanager.ha.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>sandbox.hortonworks.com</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
+ <name>yarn.resourcemanager.nodes.exclude-path</name>
+ <value>/etc/hadoop/conf/yarn.exclude</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
+ <name>yarn.resourcemanager.recovery.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>false</value>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>sandbox.hortonworks.com:8025</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>sandbox.hortonworks.com:8030</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+ <name>yarn.resourcemanager.state-store.max-completed-applications</name>
+ <value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
+ <name>yarn.resourcemanager.store.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
+ <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+ <value>10</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>sandbox.hortonworks.com:8088</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
+ <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.zk-acl</name>
+ <value>world:anyone:rwcda</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>localhost:2181</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.zk-num-retries</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>false</value>
+ <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
+ <name>yarn.resourcemanager.zk-state-store.parent-path</name>
+ <value>/rmstore</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda </value>
+ <name>yarn.resourcemanager.zk-timeout-ms</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>14336</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>3584</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.timeline-service.address</name>
+ <value>sandbox.hortonworks.com:10200</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
+ <name>yarn.timeline-service.bind-host</name>
+ <value>0.0.0.0</value>
</property>
-
+
<property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
+ <name>yarn.timeline-service.client.max-retries</name>
+ <value>30</value>
</property>
-
+
<property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>2250</value>
+ <name>yarn.timeline-service.client.retry-interval-ms</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>250</value>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
+ <name>yarn.timeline-service.generic-application-history.store-class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
+ <value>104857600</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
+ <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
+ <value>300000</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
+ <name>yarn.timeline-service.store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
+ <name>yarn.timeline-service.ttl-enable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
+ <name>yarn.timeline-service.ttl-ms</name>
+ <value>2678400000</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
- <value>10000</value>
+ <name>yarn.timeline-service.webapp.address</name>
+ <value>sandbox.hortonworks.com:8188</value>
</property>
-
+
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ <value>sandbox.hortonworks.com:8190</value>
</property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
- </configuration>
+
+</configuration>
[06/32] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
new file mode 100644
index 0000000..492b9bf
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITIIQueryTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import com.google.common.collect.Maps;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.routing.RoutingRules.RealizationPriorityRule;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ */
+@RunWith(Parameterized.class)
+public class ITIIQueryTest extends ITKylinQueryTest {
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ // give II higher priority than other realizations
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 0);
+ priorities.put(RealizationType.CUBE, 1);
+ priorities.put(RealizationType.HYBRID, 1);
+ RealizationPriorityRule.setPriorities(priorities);
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ ITKylinQueryTest.tearDown();//invoke super class
+
+ Map<RealizationType, Integer> priorities = Maps.newHashMap();
+ priorities.put(RealizationType.INVERTED_INDEX, 1);
+ priorities.put(RealizationType.CUBE, 0);
+ priorities.put(RealizationType.HYBRID, 0);
+ RealizationPriorityRule.setPriorities(priorities);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> configs() {
+ return Arrays.asList(new Object[][] { { "inner" }, { "left" } });
+ }
+
+ public ITIIQueryTest(String joinType) throws Exception {
+
+ ITKylinQueryTest.clean();
+
+ ITKylinQueryTest.joinType = joinType;
+ ITKylinQueryTest.setupAll();
+
+ }
+
+ @Test
+ public void testSingleRunQuery() throws Exception {
+ super.testSingleRunQuery();
+ }
+
+ @Test
+ public void testDetailedQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_ii", null, true);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
new file mode 100644
index 0000000..5e14471
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.test;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.enumerator.OLAPQuery;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.dbunit.database.DatabaseConnection;
+import org.dbunit.database.IDatabaseConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.File;
+import java.sql.DriverManager;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
+public class ITKylinQueryTest extends KylinTestBase {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ printInfo("setUp in KylinQueryTest");
+ joinType = "left";
+
+ setupAll();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ printInfo("tearDown");
+ printInfo("Closing connection...");
+ clean();
+ }
+
+ protected static void setupAll() throws Exception {
+ //setup env
+ HBaseMetadataTestCase.staticCreateTestMetadata();
+ config = KylinConfig.getInstanceFromEnv();
+
+ //setup cube conn
+ File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
+ Properties props = new Properties();
+ props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
+ cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
+
+ //setup h2
+ h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", "");
+ // Load H2 Tables (inner join)
+ H2Database h2DB = new H2Database(h2Connection, config);
+ h2DB.loadAllTables(joinType);
+ }
+
+ protected static void clean() {
+ if (cubeConnection != null)
+ closeConnection(cubeConnection);
+ if (h2Connection != null)
+ closeConnection(h2Connection);
+
+ ObserverEnabler.forceCoprocessorUnset();
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Ignore("this is only for debug")
+ @Test
+ public void testTempQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/temp", null, true);
+ }
+
+ @Test
+ public void testSingleRunQuery() throws Exception {
+
+ String queryFileName = "src/test/resources/query/sql/query00.sql";
+
+ File sqlFile = new File(queryFileName);
+ if (sqlFile.exists()) {
+ runSQL(sqlFile, true, true);
+ runSQL(sqlFile, true, false);
+ }
+ }
+
+ @Test
+ public void testSingleExecuteQuery() throws Exception {
+
+ String queryFileName = "src/test/resources/query/sql/query53.sql";
+
+ File sqlFile = new File(queryFileName);
+ String sql = getTextFromFile(sqlFile);
+ IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
+
+ executeQuery(kylinConn, queryFileName, sql, true);
+ }
+
+ @Ignore
+ @Test
+ public void testTableauProbing() throws Exception {
+ batchExecuteQuery("src/test/resources/query/tableau_probing");
+ }
+
+ @Test
+ public void testCommonQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql", null, true);
+ }
+
+ @Ignore
+ @Test
+ public void testSimpleQuery() throws Exception {
+ verifyResultRowCount("src/test/resources/query/sql_verifyCount");
+ }
+
+ @Test
+ public void testOrderByQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_orderby", null, true);
+ // FIXME
+ // as of optiq 0.8, we lost metadata type with "order by" clause, e.g. sql_orderby/query01.sql
+ // thus, temporarily the "order by" clause was cross out, and the needSort is set to true
+ // execAndCompQuery("src/test/resources/query/sql_orderby", null, false);
+ }
+
+ @Test
+ public void testLookupQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_lookup", null, true);
+ }
+
+ @Test
+ public void testCachedQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_cache", null, true);
+ }
+
+ @Test
+ public void testDerivedColumnQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_derived", null, true);
+ }
+
+ @Test
+ public void testDistinctCountQuery() throws Exception {
+ batchExecuteQuery("src/test/resources/query/sql_distinct");
+ }
+
+ @Test
+ public void testTableauQuery() throws Exception {
+ batchExecuteQuery("src/test/resources/query/sql_tableau");
+ }
+
+ @Test
+ public void testSubQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_subquery", null, true);
+ }
+
+ @Test
+ public void testCaseWhen() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_casewhen", null, true);
+ }
+
+ @Ignore
+ @Test
+ public void testHiveQuery() throws Exception {
+ execAndCompQuery("src/test/resources/query/sql_hive", null, true);
+ }
+
+ @Test
+ public void testH2Query() throws Exception {
+ this.execQueryUsingH2("src/test/resources/query/sql_orderby", false);
+ }
+
+ @Test
+ public void testInvalidQuery() throws Exception {
+
+ printInfo("-------------------- Test Invalid Query --------------------");
+ String queryFolder = "src/test/resources/query/sql_invalid";
+ List<File> sqlFiles = getFilesFromFolder(new File(queryFolder), ".sql");
+ for (File sqlFile : sqlFiles) {
+ String queryName = StringUtils.split(sqlFile.getName(), '.')[0];
+ printInfo("Testing Query " + queryName);
+ String sql = getTextFromFile(sqlFile);
+ IDatabaseConnection cubeConn = new DatabaseConnection(cubeConnection);
+ try {
+ cubeConn.createQueryTable(queryName, sql);
+ } catch (Throwable t) {
+ continue;
+ } finally {
+ cubeConn.close();
+ }
+ throw new IllegalStateException(queryName + " should be error!");
+ }
+ }
+
+ @Test
+ public void testDynamicQuery() throws Exception {
+ execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);
+ }
+
+ @Ignore("simple query will be supported by ii")
+ @Test
+ public void testLimitEnabled() throws Exception {
+ runSqlFile("src/test/resources/query/sql_optimize/enable-limit01.sql");
+ assertLimitWasEnabled();
+ }
+
+ private void assertLimitWasEnabled() {
+ OLAPContext context = getFirstOLAPContext();
+ assertTrue(context.storageContext.isLimitEnabled());
+ }
+
+ private OLAPContext getFirstOLAPContext() {
+ return OLAPContext.getThreadLocalContexts().iterator().next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
deleted file mode 100644
index cea8ee2..0000000
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.test;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.query.enumerator.OLAPQuery;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
-import org.dbunit.database.DatabaseConnection;
-import org.dbunit.database.IDatabaseConnection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.File;
-import java.sql.DriverManager;
-import java.util.List;
-import java.util.Properties;
-
-import static org.junit.Assert.assertTrue;
-
-@Ignore("KylinQueryTest is contained by CombinationTest")
-public class KylinQueryTest extends KylinTestBase {
-
- @BeforeClass
- public static void setUp() throws Exception {
- printInfo("setUp in KylinQueryTest");
- joinType = "left";
-
- setupAll();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- printInfo("tearDown");
- printInfo("Closing connection...");
- clean();
- }
-
- protected static void setupAll() throws Exception {
- //setup env
- HBaseMetadataTestCase.staticCreateTestMetadata();
- config = KylinConfig.getInstanceFromEnv();
-
- //setup cube conn
- File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config);
- Properties props = new Properties();
- props.setProperty(OLAPQuery.PROP_SCAN_THRESHOLD, "10000");
- cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props);
-
- //setup h2
- h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", "");
- // Load H2 Tables (inner join)
- H2Database h2DB = new H2Database(h2Connection, config);
- h2DB.loadAllTables(joinType);
- }
-
- protected static void clean() {
- if (cubeConnection != null)
- closeConnection(cubeConnection);
- if (h2Connection != null)
- closeConnection(h2Connection);
-
- ObserverEnabler.forceCoprocessorUnset();
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Ignore("this is only for debug")
- @Test
- public void testTempQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/temp", null, true);
- }
-
- @Test
- public void testSingleRunQuery() throws Exception {
-
- String queryFileName = "src/test/resources/query/sql/query00.sql";
-
- File sqlFile = new File(queryFileName);
- if (sqlFile.exists()) {
- runSQL(sqlFile, true, true);
- runSQL(sqlFile, true, false);
- }
- }
-
- @Test
- public void testSingleExecuteQuery() throws Exception {
-
- String queryFileName = "src/test/resources/query/sql/query53.sql";
-
- File sqlFile = new File(queryFileName);
- String sql = getTextFromFile(sqlFile);
- IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
-
- executeQuery(kylinConn, queryFileName, sql, true);
- }
-
- @Ignore
- @Test
- public void testTableauProbing() throws Exception {
- batchExecuteQuery("src/test/resources/query/tableau_probing");
- }
-
- @Test
- public void testCommonQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql", null, true);
- }
-
- @Ignore
- @Test
- public void testSimpleQuery() throws Exception {
- verifyResultRowCount("src/test/resources/query/sql_verifyCount");
- }
-
- @Test
- public void testOrderByQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_orderby", null, true);
- // FIXME
- // as of optiq 0.8, we lost metadata type with "order by" clause, e.g. sql_orderby/query01.sql
- // thus, temporarily the "order by" clause was cross out, and the needSort is set to true
- // execAndCompQuery("src/test/resources/query/sql_orderby", null, false);
- }
-
- @Test
- public void testLookupQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_lookup", null, true);
- }
-
- @Test
- public void testCachedQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_cache", null, true);
- }
-
- @Test
- public void testDerivedColumnQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_derived", null, true);
- }
-
- @Test
- public void testDistinctCountQuery() throws Exception {
- batchExecuteQuery("src/test/resources/query/sql_distinct");
- }
-
- @Test
- public void testTableauQuery() throws Exception {
- batchExecuteQuery("src/test/resources/query/sql_tableau");
- }
-
- @Test
- public void testSubQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_subquery", null, true);
- }
-
- @Test
- public void testCaseWhen() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_casewhen", null, true);
- }
-
- @Ignore
- @Test
- public void testHiveQuery() throws Exception {
- execAndCompQuery("src/test/resources/query/sql_hive", null, true);
- }
-
- @Test
- public void testH2Query() throws Exception {
- this.execQueryUsingH2("src/test/resources/query/sql_orderby", false);
- }
-
- @Test
- public void testInvalidQuery() throws Exception {
-
- printInfo("-------------------- Test Invalid Query --------------------");
- String queryFolder = "src/test/resources/query/sql_invalid";
- List<File> sqlFiles = getFilesFromFolder(new File(queryFolder), ".sql");
- for (File sqlFile : sqlFiles) {
- String queryName = StringUtils.split(sqlFile.getName(), '.')[0];
- printInfo("Testing Query " + queryName);
- String sql = getTextFromFile(sqlFile);
- IDatabaseConnection cubeConn = new DatabaseConnection(cubeConnection);
- try {
- cubeConn.createQueryTable(queryName, sql);
- } catch (Throwable t) {
- continue;
- } finally {
- cubeConn.close();
- }
- throw new IllegalStateException(queryName + " should be error!");
- }
- }
-
- @Test
- public void testDynamicQuery() throws Exception {
- execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);
- }
-
- @Ignore("simple query will be supported by ii")
- @Test
- public void testLimitEnabled() throws Exception {
- runSqlFile("src/test/resources/query/sql_optimize/enable-limit01.sql");
- assertLimitWasEnabled();
- }
-
- private void assertLimitWasEnabled() {
- OLAPContext context = getFirstOLAPContext();
- assertTrue(context.storageContext.isLimitEnabled());
- }
-
- private OLAPContext getFirstOLAPContext() {
- return OLAPContext.getThreadLocalContexts().iterator().next();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
new file mode 100644
index 0000000..9bdb510
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -0,0 +1,245 @@
+package org.apache.kylin.jdbc;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.junit.*;
+
+import java.io.File;
+import java.sql.*;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ */
+public class ITJDBCDriverTest extends HBaseMetadataTestCase {
+
+ private static Server server = null;
+
+ private static String previousSpringProfile = null;
+
+ private static String SPRING_PROFILE_PROPERTY = "spring.profiles.active";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ previousSpringProfile = System.getProperty(SPRING_PROFILE_PROPERTY);
+ System.setProperty(SPRING_PROFILE_PROPERTY, "testing");
+ staticCreateTestMetadata();
+ startJetty();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ stopJetty();
+ staticCleanupTestMetadata();
+ if (previousSpringProfile == null) {
+ System.clearProperty(SPRING_PROFILE_PROPERTY);
+ } else {
+ System.setProperty(SPRING_PROFILE_PROPERTY, previousSpringProfile);
+ }
+ }
+
+ protected static void stopJetty() throws Exception {
+ if (server != null)
+ server.stop();
+
+ File workFolder = new File("work");
+ if(workFolder.isDirectory() && workFolder.exists()) {
+ FileUtils.deleteDirectory(workFolder);
+ }
+ }
+
+ protected static void startJetty() throws Exception {
+
+ server = new Server(7070);
+
+ WebAppContext context = new WebAppContext();
+ context.setDescriptor("./src/main/webapp/WEB-INF/web.xml");
+ context.setResourceBase("./src/main/webapp");
+ context.setContextPath("/kylin");
+ context.setParentLoaderPriority(true);
+
+ server.setHandler(context);
+
+ server.start();
+
+ }
+
+ protected Connection getConnection() throws Exception {
+
+ Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
+ Properties info = new Properties();
+ info.put("user", "ADMIN");
+ info.put("password", "KYLIN");
+ Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
+
+ return conn;
+ }
+
+ @Test
+ public void testMetadata1() throws Exception {
+
+ // check the JDBC API here: http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html
+ Connection conn = getConnection();
+
+ // test getSchemas();
+ List<String> schemaList = Lists.newArrayList();
+ DatabaseMetaData dbMetadata = conn.getMetaData();
+ ResultSet resultSet = dbMetadata.getSchemas();
+ while (resultSet.next()) {
+ String schema = resultSet.getString("TABLE_SCHEM");
+ String catalog = resultSet.getString("TABLE_CATALOG");
+
+ System.out.println("Get schema: schema=" + schema + ", catalog=" + catalog);
+ schemaList.add(schema);
+
+ }
+
+ resultSet.close();
+ Assert.assertTrue(schemaList.contains("DEFAULT"));
+ Assert.assertTrue(schemaList.contains("EDW"));
+
+ // test getCatalogs();
+ resultSet = dbMetadata.getCatalogs();
+
+ List<String> catalogList = Lists.newArrayList();
+ while (resultSet.next()) {
+ String catalog = resultSet.getString("TABLE_CAT");
+
+ System.out.println("Get catalog: catalog=" + catalog);
+ catalogList.add(catalog);
+
+ }
+ Assert.assertTrue(catalogList.size() > 0 && catalogList.contains("defaultCatalog"));
+
+ /** //Disable the test on getTableTypes() as it is not ready
+ resultSet = dbMetadata.getTableTypes();
+
+ List<String> tableTypes = Lists.newArrayList();
+ while (resultSet.next()) {
+ String type = resultSet.getString("TABLE_TYPE");
+
+ System.out.println("Get table type: type=" + type);
+ tableTypes.add(type);
+
+ }
+
+ Assert.assertTrue(tableTypes.size() > 0 && tableTypes.contains("TABLE"));
+ resultSet.close();
+
+ **/
+ conn.close();
+ }
+
+ @Test
+ public void testMetadata2() throws Exception {
+ Connection conn = getConnection();
+
+ List<String> tableList = Lists.newArrayList();
+ DatabaseMetaData dbMetadata = conn.getMetaData();
+ ResultSet resultSet = dbMetadata.getTables(null, "%", "%", new String[]{"TABLE"});
+ while (resultSet.next()) {
+ String schema = resultSet.getString("TABLE_SCHEM");
+ String name = resultSet.getString("TABLE_NAME");
+
+ System.out.println("Get table: schema=" + schema + ", name=" + name);
+ tableList.add(schema + "." + name);
+
+ }
+
+ resultSet.close();
+ Assert.assertTrue(tableList.contains("DEFAULT.TEST_KYLIN_FACT"));
+
+ resultSet = dbMetadata.getColumns(null, "%", "TEST_KYLIN_FACT", "%");
+
+ List<String> columns = Lists.newArrayList();
+ while (resultSet.next()) {
+ String name = resultSet.getString("COLUMN_NAME");
+ String type = resultSet.getString("TYPE_NAME");
+
+ System.out.println("Get column: name=" + name + ", data_type=" + type);
+ columns.add(name);
+
+ }
+
+ Assert.assertTrue(columns.size() > 0 && columns.contains("CAL_DT"));
+ resultSet.close();
+ conn.close();
+ }
+
+ @Test
+ public void testSimpleStatement() throws Exception {
+ Connection conn = getConnection();
+ Statement statement = conn.createStatement();
+
+ statement.execute("select count(*) from test_kylin_fact");
+
+ ResultSet rs = statement.getResultSet();
+
+ Assert.assertTrue(rs.next());
+ int result = rs.getInt(1);
+
+ Assert.assertTrue(result > 0);
+
+ rs.close();
+ statement.close();
+ conn.close();
+
+ }
+
+
+ @Test
+ public void testPreparedStatement() throws Exception {
+ Connection conn = getConnection();
+
+ PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " +
+ "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
+
+ statement.setString(1, "FP-GTC");
+
+ ResultSet rs = statement.executeQuery();
+
+ Assert.assertTrue(rs.next());
+
+ String format_name = rs.getString(1);
+
+ Assert.assertTrue("FP-GTC".equals(format_name));
+
+ rs.close();
+ statement.close();
+ conn.close();
+
+ }
+
+ @Test
+ public void testResultSet() throws Exception {
+ String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" +
+ " group by LSTG_FORMAT_NAME ";
+
+ Connection conn = getConnection();
+ Statement statement = conn.createStatement();
+
+ statement.execute(sql);
+
+ ResultSet rs = statement.getResultSet();
+
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ String lstg = rs.getString(1);
+ double gmv = rs.getDouble(2);
+ int trans_count = rs.getInt(3);
+
+ System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count);
+ }
+
+ Assert.assertTrue(count > 0);
+ statement.close();
+ rs.close();
+ conn.close();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
deleted file mode 100644
index 2eff08f..0000000
--- a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-package org.apache.kylin.jdbc;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.io.FileUtils;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.junit.*;
-
-import java.io.File;
-import java.sql.*;
-import java.util.List;
-import java.util.Properties;
-
-/**
- */
-public class JDBCDriverTest extends HBaseMetadataTestCase {
-
- private static Server server = null;
-
- private static String previousSpringProfile = null;
-
- private static String SPRING_PROFILE_PROPERTY = "spring.profiles.active";
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- previousSpringProfile = System.getProperty(SPRING_PROFILE_PROPERTY);
- System.setProperty(SPRING_PROFILE_PROPERTY, "testing");
- staticCreateTestMetadata();
- startJetty();
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- stopJetty();
- staticCleanupTestMetadata();
- if (previousSpringProfile == null) {
- System.clearProperty(SPRING_PROFILE_PROPERTY);
- } else {
- System.setProperty(SPRING_PROFILE_PROPERTY, previousSpringProfile);
- }
- }
-
- protected static void stopJetty() throws Exception {
- if (server != null)
- server.stop();
-
- File workFolder = new File("work");
- if(workFolder.isDirectory() && workFolder.exists()) {
- FileUtils.deleteDirectory(workFolder);
- }
- }
-
- protected static void startJetty() throws Exception {
-
- server = new Server(7070);
-
- WebAppContext context = new WebAppContext();
- context.setDescriptor("./src/main/webapp/WEB-INF/web.xml");
- context.setResourceBase("./src/main/webapp");
- context.setContextPath("/kylin");
- context.setParentLoaderPriority(true);
-
- server.setHandler(context);
-
- server.start();
-
- }
-
- protected Connection getConnection() throws Exception {
-
- Driver driver = (Driver) Class.forName("org.apache.kylin.jdbc.Driver").newInstance();
- Properties info = new Properties();
- info.put("user", "ADMIN");
- info.put("password", "KYLIN");
- Connection conn = driver.connect("jdbc:kylin://localhost:7070/default", info);
-
- return conn;
- }
-
- @Test
- public void testMetadata1() throws Exception {
-
- // check the JDBC API here: http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html
- Connection conn = getConnection();
-
- // test getSchemas();
- List<String> schemaList = Lists.newArrayList();
- DatabaseMetaData dbMetadata = conn.getMetaData();
- ResultSet resultSet = dbMetadata.getSchemas();
- while (resultSet.next()) {
- String schema = resultSet.getString("TABLE_SCHEM");
- String catalog = resultSet.getString("TABLE_CATALOG");
-
- System.out.println("Get schema: schema=" + schema + ", catalog=" + catalog);
- schemaList.add(schema);
-
- }
-
- resultSet.close();
- Assert.assertTrue(schemaList.contains("DEFAULT"));
- Assert.assertTrue(schemaList.contains("EDW"));
-
- // test getCatalogs();
- resultSet = dbMetadata.getCatalogs();
-
- List<String> catalogList = Lists.newArrayList();
- while (resultSet.next()) {
- String catalog = resultSet.getString("TABLE_CAT");
-
- System.out.println("Get catalog: catalog=" + catalog);
- catalogList.add(catalog);
-
- }
- Assert.assertTrue(catalogList.size() > 0 && catalogList.contains("defaultCatalog"));
-
- /** //Disable the test on getTableTypes() as it is not ready
- resultSet = dbMetadata.getTableTypes();
-
- List<String> tableTypes = Lists.newArrayList();
- while (resultSet.next()) {
- String type = resultSet.getString("TABLE_TYPE");
-
- System.out.println("Get table type: type=" + type);
- tableTypes.add(type);
-
- }
-
- Assert.assertTrue(tableTypes.size() > 0 && tableTypes.contains("TABLE"));
- resultSet.close();
-
- **/
- conn.close();
- }
-
- @Test
- public void testMetadata2() throws Exception {
- Connection conn = getConnection();
-
- List<String> tableList = Lists.newArrayList();
- DatabaseMetaData dbMetadata = conn.getMetaData();
- ResultSet resultSet = dbMetadata.getTables(null, "%", "%", new String[]{"TABLE"});
- while (resultSet.next()) {
- String schema = resultSet.getString("TABLE_SCHEM");
- String name = resultSet.getString("TABLE_NAME");
-
- System.out.println("Get table: schema=" + schema + ", name=" + name);
- tableList.add(schema + "." + name);
-
- }
-
- resultSet.close();
- Assert.assertTrue(tableList.contains("DEFAULT.TEST_KYLIN_FACT"));
-
- resultSet = dbMetadata.getColumns(null, "%", "TEST_KYLIN_FACT", "%");
-
- List<String> columns = Lists.newArrayList();
- while (resultSet.next()) {
- String name = resultSet.getString("COLUMN_NAME");
- String type = resultSet.getString("TYPE_NAME");
-
- System.out.println("Get column: name=" + name + ", data_type=" + type);
- columns.add(name);
-
- }
-
- Assert.assertTrue(columns.size() > 0 && columns.contains("CAL_DT"));
- resultSet.close();
- conn.close();
- }
-
- @Test
- public void testSimpleStatement() throws Exception {
- Connection conn = getConnection();
- Statement statement = conn.createStatement();
-
- statement.execute("select count(*) from test_kylin_fact");
-
- ResultSet rs = statement.getResultSet();
-
- Assert.assertTrue(rs.next());
- int result = rs.getInt(1);
-
- Assert.assertTrue(result > 0);
-
- rs.close();
- statement.close();
- conn.close();
-
- }
-
-
- @Test
- public void testPreparedStatement() throws Exception {
- Connection conn = getConnection();
-
- PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " +
- "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
-
- statement.setString(1, "FP-GTC");
-
- ResultSet rs = statement.executeQuery();
-
- Assert.assertTrue(rs.next());
-
- String format_name = rs.getString(1);
-
- Assert.assertTrue("FP-GTC".equals(format_name));
-
- rs.close();
- statement.close();
- conn.close();
-
- }
-
- @Test
- public void testResultSet() throws Exception {
- String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" +
- " group by LSTG_FORMAT_NAME ";
-
- Connection conn = getConnection();
- Statement statement = conn.createStatement();
-
- statement.execute(sql);
-
- ResultSet rs = statement.getResultSet();
-
- int count = 0;
- while (rs.next()) {
- count++;
- String lstg = rs.getString(1);
- double gmv = rs.getDouble(2);
- int trans_count = rs.getInt(3);
-
- System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count);
- }
-
- Assert.assertTrue(count > 0);
- statement.close();
- rs.close();
- conn.close();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
new file mode 100644
index 0000000..75defbf
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ITTableControllerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shaoshi
+ */
+public class ITTableControllerTest extends ServiceTestBase {
+
+ private TableController tableController;
+ private CubeDescController cubeDescController;
+
+ @Autowired
+ CubeService cubeService;
+
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+
+ tableController = new TableController();
+ tableController.setCubeService(cubeService);
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ List<TableDesc> tables = tableController.getHiveTables(true, "default");
+
+ Assert.assertTrue(tables != null && tables.size() > 0);
+
+ TableDesc factTable = null;
+ for (TableDesc t : tables) {
+ if (t.getName().equalsIgnoreCase("test_kylin_fact")) {
+ factTable = t;
+ break;
+ }
+ }
+
+ Assert.assertNotNull(factTable);
+
+
+ Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
+ Assert.assertNotNull(loadResult);
+
+ Assert.assertTrue(loadResult.get("result.loaded").length ==2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
deleted file mode 100644
index 4437abe..0000000
--- a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.controller;
-
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.rest.service.CubeService;
-import org.apache.kylin.rest.service.ServiceTestBase;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author shaoshi
- */
-public class TableControllerTest extends ServiceTestBase {
-
- private TableController tableController;
- private CubeDescController cubeDescController;
-
- @Autowired
- CubeService cubeService;
-
- @Before
- public void setup() throws Exception {
- super.setup();
-
- tableController = new TableController();
- tableController.setCubeService(cubeService);
- }
-
- @Test
- public void testBasics() throws IOException {
- List<TableDesc> tables = tableController.getHiveTables(true, "default");
-
- Assert.assertTrue(tables != null && tables.size() > 0);
-
- TableDesc factTable = null;
- for (TableDesc t : tables) {
- if (t.getName().equalsIgnoreCase("test_kylin_fact")) {
- factTable = t;
- break;
- }
- }
-
- Assert.assertNotNull(factTable);
-
-
- Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
- Assert.assertNotNull(loadResult);
-
- Assert.assertTrue(loadResult.get("result.loaded").length ==2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 559ec89..9f0598d 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -129,6 +129,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@After
public void after() throws Exception {
+ cleanupTestMetadata();
}
private void waitForCounterAndClear(long count) {
@@ -145,16 +146,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
}
}
- @BeforeClass
- public static void startServer() throws Exception {
-
- }
-
- @AfterClass
- public static void stopServer() throws Exception {
-
- }
-
private static CubeManager getCubeManager(KylinConfig config) throws Exception {
return CubeManager.getInstance(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
new file mode 100644
index 0000000..c92921b
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/ITInvertedIndexHBaseTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecord;
+import org.apache.kylin.invertedindex.index.TableRecordInfo;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * @author yangli9
+ */
+public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase {
+
+ IIInstance ii;
+ IISegment seg;
+ HConnection hconn;
+
+ TableRecordInfo info;
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+
+ this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
+ this.seg = ii.getFirstSegment();
+
+ String hbaseUrl = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
+ hconn = HConnectionManager.createConnection(hconf);
+
+ this.info = new TableRecordInfo(seg);
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testLoad() throws Exception {
+
+ String tableName = seg.getStorageLocationIdentifier();
+ IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
+
+ List<Slice> slices = Lists.newArrayList();
+ HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
+ try {
+ for (Slice slice : codec.decodeKeyValue(kvIterator)) {
+ slices.add(slice);
+ }
+ } finally {
+ kvIterator.close();
+ }
+
+ List<TableRecord> records = iterateRecords(slices);
+ //dump(records);
+ System.out.println(records.size() + " records");
+ }
+
+ private List<TableRecord> iterateRecords(List<Slice> slices) {
+ List<TableRecord> records = Lists.newArrayList();
+ for (Slice slice : slices) {
+ for (RawTableRecord rec : slice) {
+ records.add(new TableRecord((RawTableRecord) rec.clone(), info));
+ }
+ }
+ return records;
+ }
+
+ private void dump(Iterable<TableRecord> records) {
+ for (TableRecord rec : records) {
+ byte[] x = rec.getBytes();
+ String y = BytesUtil.toReadableText(x);
+ System.out.println(y);
+ System.out.println();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
deleted file mode 100644
index fd24487..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.index.RawTableRecord;
-import org.apache.kylin.invertedindex.index.Slice;
-import org.apache.kylin.invertedindex.index.TableRecord;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.List;
-
-/**
- * @author yangli9
- */
-public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
-
- IIInstance ii;
- IISegment seg;
- HConnection hconn;
-
- TableRecordInfo info;
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
-
- this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");
- this.seg = ii.getFirstSegment();
-
- String hbaseUrl = KylinConfig.getInstanceFromEnv().getStorageUrl();
- Configuration hconf = HadoopUtil.newHBaseConfiguration(hbaseUrl);
- hconn = HConnectionManager.createConnection(hconf);
-
- this.info = new TableRecordInfo(seg);
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testLoad() throws Exception {
-
- String tableName = seg.getStorageLocationIdentifier();
- IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest());
-
- List<Slice> slices = Lists.newArrayList();
- HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES);
- try {
- for (Slice slice : codec.decodeKeyValue(kvIterator)) {
- slices.add(slice);
- }
- } finally {
- kvIterator.close();
- }
-
- List<TableRecord> records = iterateRecords(slices);
- //dump(records);
- System.out.println(records.size() + " records");
- }
-
- private List<TableRecord> iterateRecords(List<Slice> slices) {
- List<TableRecord> records = Lists.newArrayList();
- for (Slice slice : slices) {
- for (RawTableRecord rec : slice) {
- records.add(new TableRecord((RawTableRecord) rec.clone(), info));
- }
- }
- return records;
- }
-
- private void dump(Iterable<TableRecord> records) {
- for (TableRecord rec : records) {
- byte[] x = rec.getBytes();
- String y = BytesUtil.toReadableText(x);
- System.out.println(y);
- System.out.println();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
new file mode 100644
index 0000000..1e32ef5
--- /dev/null
+++ b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.test;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageEngineFactory;
+import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
+import org.junit.*;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+public class ITStorageTest extends HBaseMetadataTestCase {
+
+ private IStorageEngine storageEngine;
+ private CubeInstance cube;
+ private StorageContext context;
+
+ @BeforeClass
+ public static void setupResource() throws Exception {
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+
+ CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
+ cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
+ Assert.assertNotNull(cube);
+ storageEngine = StorageEngineFactory.getStorageEngine(cube);
+ String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
+ context = new StorageContext();
+ context.setConnUrl(url);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test(expected = ScanOutOfLimitException.class)
+ @Ignore
+ public void testScanOutOfLimit() {
+ context.setThreshold(1);
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+
+ search(groups, aggregations, null, context);
+ }
+
+ @Test
+ public void test01() {
+ List<TblColRef> groups = StorageMockUtils.buildGroups();
+ List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ /*
+ @Test
+ public void test02() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildFilter2(groups.get(1));
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test03() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildAndFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test04() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+ TupleFilter filter = buildOrFilter(groups);
+
+ int count = search(groups, aggregations, filter, context);
+ assertTrue(count > 0);
+ }
+
+ @Test
+ public void test05() {
+ List<TblColRef> groups = buildGroups();
+ List<FunctionDesc> aggregations = buildAggregations();
+
+ int count = search(groups, aggregations, null, context);
+ assertTrue(count > 0);
+ }
+ */
+ private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
+ int count = 0;
+ ITupleIterator iterator = null;
+ try {
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
+ iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
+ while (iterator.hasNext()) {
+ ITuple tuple = iterator.next();
+ System.out.println("Tuple = " + tuple);
+ count++;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (iterator != null)
+ iterator.close();
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
deleted file mode 100644
index f9a6212..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.test;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageEngine;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageEngineFactory;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
-import org.junit.*;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertTrue;
-
-public class StorageTest extends HBaseMetadataTestCase {
-
- private IStorageEngine storageEngine;
- private CubeInstance cube;
- private StorageContext context;
-
- @BeforeClass
- public static void setupResource() throws Exception {
- }
-
- @AfterClass
- public static void tearDownResource() {
- }
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
- Assert.assertNotNull(cube);
- storageEngine = StorageEngineFactory.getStorageEngine(cube);
- String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
- context = new StorageContext();
- context.setConnUrl(url);
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test(expected = ScanOutOfLimitException.class)
- @Ignore
- public void testScanOutOfLimit() {
- context.setThreshold(1);
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-
- search(groups, aggregations, null, context);
- }
-
- @Test
- public void test01() {
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- /*
- @Test
- public void test02() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildFilter2(groups.get(1));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test03() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildAndFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test04() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildOrFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test05() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
-
- int count = search(groups, aggregations, null, context);
- assertTrue(count > 0);
- }
- */
- private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
- int count = 0;
- ITupleIterator iterator = null;
- try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
- iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
- while (iterator.hasNext()) {
- ITuple tuple = iterator.next();
- System.out.println("Tuple = " + tuple);
- count++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- if (iterator != null)
- iterator.close();
- }
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
new file mode 100644
index 0000000..3008314
--- /dev/null
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
@@ -0,0 +1,108 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.streaming;
+
+import com.google.common.collect.Lists;
+import org.apache.kylin.common.KylinConfig;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ */
+public class ITKafkaConsumerTest extends KafkaBaseTest {
+
+ private OneOffStreamProducer producer;
+
+ private static final int TOTAL_SEND_COUNT = 100;
+
+ private KafkaConfig kafkaConfig;
+
+ @Before
+ public void before() throws IOException {
+ producer = new OneOffStreamProducer(TOTAL_SEND_COUNT);
+ producer.start();
+ kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
+ }
+
+ @After
+ public void after() {
+ producer.stop();
+ }
+
+ private void waitForProducerToStop(OneOffStreamProducer producer) {
+ while (!producer.isStopped()) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Test
+ @Ignore("since ci does not have the topic")
+ public void test() throws InterruptedException {
+ final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
+ final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
+ List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
+ for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
+ KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, 0, kafkaConfig.getBrokers(), kafkaConfig);
+ queues.add(consumer.getStreamQueue(0));
+ executorService.execute(consumer);
+ }
+ waitForProducerToStop(producer);
+
+ //wait some time to ensure consumer has fetched all data
+ Thread.sleep(5000);
+ int count = 0;
+ for (BlockingQueue<StreamMessage> queue : queues) {
+ count += queue.size();
+ }
+
+ logger.info("count of messages are " + count);
+ //since there will be historical data
+ assertTrue(count >= TOTAL_SEND_COUNT && (count % TOTAL_SEND_COUNT == 0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
new file mode 100644
index 0000000..d8853f6
--- /dev/null
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaRequesterTest.java
@@ -0,0 +1,76 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.streaming;
+
+import org.apache.kylin.common.KylinConfig;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ */
+public class ITKafkaRequesterTest extends KafkaBaseTest {
+
+ private static final String NON_EXISTED_TOPIC = "non_existent_topic";
+ private KafkaConfig kafkaConfig;
+
+
+ @Before
+ public void before() {
+ kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ }
+
+ @Test
+ @Ignore("since ci does not enable kafka")
+ public void testTopicMeta() throws Exception {
+ TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
+ assertNotNull(kafkaTopicMeta);
+ assertEquals(2, kafkaTopicMeta.getPartitionIds().size());
+ assertEquals(kafkaConfig.getTopic(), kafkaTopicMeta.getName());
+
+ KafkaConfig anotherTopicConfig = kafkaConfig.clone();
+ anotherTopicConfig.setTopic(NON_EXISTED_TOPIC);
+
+ kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(anotherTopicConfig);
+ assertTrue(kafkaTopicMeta == null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
deleted file mode 100644
index c782dae..0000000
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaConsumerTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.streaming;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.KylinConfig;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- */
-public class KafkaConsumerTest extends KafkaBaseTest {
-
- private OneOffStreamProducer producer;
-
- private static final int TOTAL_SEND_COUNT = 100;
-
- private KafkaConfig kafkaConfig;
-
- @Before
- public void before() throws IOException {
- producer = new OneOffStreamProducer(TOTAL_SEND_COUNT);
- producer.start();
- kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
- }
-
- @After
- public void after() {
- producer.stop();
- }
-
- private void waitForProducerToStop(OneOffStreamProducer producer) {
- while (!producer.isStopped()) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-
- @Test
- @Ignore("since ci does not have the topic")
- public void test() throws InterruptedException {
- final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
- final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
- List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
- for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
- KafkaConsumer consumer = new KafkaConsumer(kafkaTopicMeta.getName(), partitionId, 0, kafkaConfig.getBrokers(), kafkaConfig);
- queues.add(consumer.getStreamQueue(0));
- executorService.execute(consumer);
- }
- waitForProducerToStop(producer);
-
- //wait some time to ensure consumer has fetched all data
- Thread.sleep(5000);
- int count = 0;
- for (BlockingQueue<StreamMessage> queue : queues) {
- count += queue.size();
- }
-
- logger.info("count of messages are " + count);
- //since there will be historical data
- assertTrue(count >= TOTAL_SEND_COUNT && (count % TOTAL_SEND_COUNT == 0));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java b/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
deleted file mode 100644
index 9d99825..0000000
--- a/streaming/src/test/java/org/apache/kylin/streaming/KafkaRequesterTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.streaming;
-
-import org.apache.kylin.common.KylinConfig;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- */
-public class KafkaRequesterTest extends KafkaBaseTest {
-
- private static final String NON_EXISTED_TOPIC = "non_existent_topic";
- private KafkaConfig kafkaConfig;
-
-
- @Before
- public void before() {
- kafkaConfig = StreamingManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig("kafka_test");
- }
-
- @AfterClass
- public static void afterClass() {
- }
-
- @Test
- @Ignore("since ci does not enable kafka")
- public void testTopicMeta() throws Exception {
- TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaConfig);
- assertNotNull(kafkaTopicMeta);
- assertEquals(2, kafkaTopicMeta.getPartitionIds().size());
- assertEquals(kafkaConfig.getTopic(), kafkaTopicMeta.getName());
-
- KafkaConfig anotherTopicConfig = kafkaConfig.clone();
- anotherTopicConfig.setTopic(NON_EXISTED_TOPIC);
-
- kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(anotherTopicConfig);
- assertTrue(kafkaTopicMeta == null);
- }
-}
[24/32] incubator-kylin git commit: KYLIN-783 change hdp.version to
2.2.4.2-2
Posted by ma...@apache.org.
KYLIN-783 change hdp.version to 2.2.4.2-2
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/69c1a939
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/69c1a939
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/69c1a939
Branch: refs/heads/0.8.0
Commit: 69c1a939299dae8d1a8b66c081d496878942cfa3
Parents: 84e4d7f
Author: honma <ho...@ebay.com>
Authored: Tue May 26 11:20:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../org/apache/kylin/job/BuildCubeWithStreamTest.java | 2 +-
.../java/org/apache/kylin/job/BuildIIWithEngineTest.java | 2 +-
.../java/org/apache/kylin/job/BuildIIWithStreamTest.java | 2 +-
.../kylin/job/ITKafkaBasedIIStreamBuilderTest.java | 2 +-
pom.xml | 2 +-
.../src/main/java/org/apache/kylin/rest/DebugTomcat.java | 11 +++++------
7 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 547a6e7..d3c3669 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -81,7 +81,7 @@ public class BuildCubeWithEngineTest {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index dc42dc5..a437bba 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -91,7 +91,7 @@ public class BuildCubeWithStreamTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 9c0219a..612ba6c 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -84,7 +84,7 @@ public class BuildIIWithEngineTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 6bceab7..9876ace 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -92,7 +92,7 @@ public class BuildIIWithStreamTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
index d7aa856..2432220 100644
--- a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -60,7 +60,7 @@ public class ITKafkaBasedIIStreamBuilderTest {
@BeforeClass
public static void beforeClass() throws Exception {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e522050..21fbed8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -615,7 +615,7 @@
</property>
<property>
<name>hdp.version</name>
- <value>2.2.0.0-2041</value>
+ <value>2.2.4.2-2</value>
</property>
</systemProperties>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69c1a939/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index a1cc86b..0b5db67 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -18,20 +18,19 @@
package org.apache.kylin.rest;
-import java.io.File;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
import org.apache.hadoop.util.Shell;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.util.ClasspathUtil;
+import java.io.File;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
public class DebugTomcat {
public static void setupDebugEnv() {
@@ -39,7 +38,7 @@ public class DebugTomcat {
// test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
System.setProperty("spring.profiles.active", "testing");
// workaround for job submission from win to linux -- https://issues.apache.org/jira/browse/MAPREDUCE-4052
[17/32] incubator-kylin git commit: KYLIN-697 remove duplicate
failsafe definition
Posted by ma...@apache.org.
KYLIN-697 remove duplicate failsafe definition
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7fcfada4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7fcfada4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7fcfada4
Branch: refs/heads/0.8.0
Commit: 7fcfada4f0099a53e5d7a8b2b896e5cce289b65d
Parents: ca6a99a
Author: honma <ho...@ebay.com>
Authored: Tue May 26 10:40:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
pom.xml | 31 ++++++++++++++-----------------
1 file changed, 14 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7fcfada4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 204a5a4..e522050 100644
--- a/pom.xml
+++ b/pom.xml
@@ -495,19 +495,19 @@
<includePom>true</includePom>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <goals>
- <goal>integration-test</goal>
- <goal>verify</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <!--<plugin>-->
+ <!--<groupId>org.apache.maven.plugins</groupId>-->
+ <!--<artifactId>maven-failsafe-plugin</artifactId>-->
+ <!--<version>2.6</version>-->
+ <!--<executions>-->
+ <!--<execution>-->
+ <!--<goals>-->
+ <!--<goal>integration-test</goal>-->
+ <!--<goal>verify</goal>-->
+ <!--</goals>-->
+ <!--</execution>-->
+ <!--</executions>-->
+ <!--</plugin>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
@@ -617,16 +617,13 @@
<name>hdp.version</name>
<value>2.2.0.0-2041</value>
</property>
- <property>
- <name>buildCubeUsingProvidedData</name>
- <value>false</value>
- </property>
</systemProperties>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.6</version>
<executions>
<execution>
<id>integration-tests</id>
[26/32] incubator-kylin git commit: KYLIN-693 character all build
cube/ii test cases
Posted by ma...@apache.org.
KYLIN-693 character all build cube/ii test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/29058abc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/29058abc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/29058abc
Branch: refs/heads/0.8.0
Commit: 29058abc932806769bba4d2cbbfcd3561515934b
Parents: 3a0d9d2
Author: honma <ho...@ebay.com>
Authored: Mon May 25 14:15:39 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/BuildCubeWithStreamTest.java | 224 +++++++++++++++
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 +++++++++++++++++
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 ---------------
.../kylin/job/ITBuildIIWithEngineTest.java | 254 -----------------
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ------------------
8 files changed, 1027 insertions(+), 1027 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
new file mode 100644
index 0000000..547a6e7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.cube.CubingJob;
+import org.apache.kylin.job.cube.CubingJobBuilder;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class BuildCubeWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+
+ private CubeManager cubeManager;
+
+ private DefaultScheduler scheduler;
+
+ protected ExecutableManager jobService;
+
+ private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if(jobService.getJob(jobId) instanceof CubingJob){
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
+ testInner();
+ testLeft();
+ }
+
+ private void testInner() throws Exception {
+ String[] testCase = new String[]{
+ "testInnerJoinCube",
+ "testInnerJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void testLeft() throws Exception {
+ String[] testCase = new String[]{
+ "testLeftJoinCube",
+ "testLeftJoinCube2",
+ };
+ runTestAndAssertSucceed(testCase);
+ }
+
+ private void runTestAndAssertSucceed(String[] testCase) throws Exception {
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ try {
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex);
+ throw ex;
+ }
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube2() throws Exception {
+ clearSegment("test_kylin_cube_with_slr_empty");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testInnerJoinCube() throws Exception {
+ clearSegment("test_kylin_cube_without_slr_empty");
+
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ // this cube's start date is 0, end date is 20501112000000
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+
+
+ // this cube doesn't support incremental build, always do full build
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
+ return result;
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube2() throws Exception {
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ List<String> result = Lists.newArrayList();
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
+ // this cube's start date is 0, end date is 20120601000000
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2012-06-01").getTime();
+
+ clearSegment(cubeName);
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+
+ // then submit an append job, start date is 20120601000000, end
+ // date is 20220101000000
+ dateStart = f.parse("2012-06-01").getTime();
+ dateEnd = f.parse("2022-01-01").getTime();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ @SuppressWarnings("unused") // called by reflection
+ private List<String> testLeftJoinCube() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
+ clearSegment(cubeName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
+ long dateEnd = f.parse("2050-11-12").getTime();
+
+ // this cube's start date is 0, end date is 20501112000000
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(cubeName, dateStart, dateEnd));
+ return result;
+
+ }
+
+ private void clearSegment(String cubeName) throws Exception {
+ CubeInstance cube = cubeManager.getCube(cubeName);
+ cube.getSegments().clear();
+ cubeManager.updateCube(cube,true);
+ }
+
+
+ private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
+ CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
+ CubingJob job = cubingJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
new file mode 100644
index 0000000..dc42dc5
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -0,0 +1,224 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.dict.Dictionary;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
+import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.gridtable.GTRecord;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ *
+ * This class is going to be deleted
+ */
+@Ignore("For dev testing")
+public class BuildCubeWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
+
+ private KylinConfig kylinConfig;
+ private CubeManager cubeManager;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+
+ }
+
+ @After
+ public void after() {
+ HBaseMetadataTestCase.staticCleanupTestMetadata();
+ }
+
+ @Test
+ public void test() throws Exception {
+ CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
+ final CubeDesc desc = cube.getDescriptor();
+ // cube.getSegments().clear();
+ // cubeManager.updateCube(cube);
+
+ CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
+ Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
+
+//
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = cubeSegment.getDictionary(col);
+ if (dict == null) {
+ throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
+ }
+ logger.info("Dictionary for " + col + " was put into dictionary map.");
+ dictionaryMap.put(col, cubeSegment.getDictionary(col));
+ }
+ }
+ }
+
+// final String tableName = createIntermediateTable(desc, kylinConfig, null);
+ String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
+// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
+ tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
+ logger.info("intermediate table name:" + tableName);
+
+
+ ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
+
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ Future<?> future = executorService.submit(cubeBuilder);
+
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ List<String> row;
+ int counter = 0;
+ while (reader.next()) {
+ row = reader.getRowAsList();
+ queue.put(row);
+ counter++;
+ if(counter == 200000)
+ break;
+ }
+ queue.put(new ArrayList<String>(0));
+ reader.close();
+
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ throw new IOException("Failed to build cube ", e);
+ }
+
+ logger.info("stream build finished");
+ }
+
+
+ private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
+ SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
+
+ List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
+ for (List<String> row : table) {
+ for (int i = 0; i < dimColumns.size(); i++) {
+ String cell = row.get(i);
+ valueMap.put(dimColumns.get(i), cell);
+ }
+ }
+
+ for (DimensionDesc dim : desc.getDimensions()) {
+ // dictionary
+ for (TblColRef col : dim.getColumnRefs()) {
+ if (desc.getRowkey().isUseDictionary(col)) {
+ Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
+ @Nullable
+ @Override
+ public byte[] apply(String input) {
+ if (input == null)
+ return null;
+ return input.getBytes();
+ }
+ }));
+
+ logger.info("Building dictionary for " + col);
+ dictionaryMap.put(col, dict);
+ }
+ }
+ }
+
+ }
+
+
+ class ConsoleGTRecordWriter implements IGTRecordWriter {
+
+ boolean verbose = false;
+
+ @Override
+ public void write(Long cuboidId, GTRecord record) throws IOException {
+ if (verbose)
+ System.out.println(record.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
new file mode 100644
index 0000000..9c0219a
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.invertedindex.IIJob;
+import org.apache.kylin.job.invertedindex.IIJobBuilder;
+import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.junit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.concurrent.*;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author shaoshi
+ */
+public class BuildIIWithEngineTest {
+
+ private JobEngineConfig jobEngineConfig;
+ private IIManager iiManager;
+
+ private DefaultScheduler scheduler;
+ protected ExecutableManager jobService;
+
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
+
+ private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
+
+ protected void waitForJob(String jobId) {
+ while (true) {
+ AbstractExecutable job = jobService.getJob(jobId);
+ if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
+ break;
+ } else {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ //DeployUtil.initCliWorkDir();
+ // DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ jobService = ExecutableManager.getInstance(kylinConfig);
+ scheduler = DefaultScheduler.getInstance();
+ scheduler.init(new JobEngineConfig(kylinConfig));
+ if (!scheduler.hasStarted()) {
+ throw new RuntimeException("scheduler has not been started");
+ }
+ jobEngineConfig = new JobEngineConfig(kylinConfig);
+ for (String jobId : jobService.getAllJobIds()) {
+ if (jobService.getJob(jobId) instanceof IIJob) {
+ jobService.deleteJob(jobId);
+ }
+ }
+
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : TEST_II_INSTANCES) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii, true);
+ }
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+
+ for (String iiInstance : TEST_II_INSTANCES) {
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii, true);
+ }
+ }
+ backup();
+ }
+
+ @Test
+ @Ignore
+ public void testBuildII() throws Exception {
+
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
+ ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
+ final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
+ List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ for (int i = 0; i < testCase.length; i++) {
+ tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
+ }
+ countDownLatch.await();
+ for (int i = 0; i < tasks.size(); ++i) {
+ Future<List<String>> task = tasks.get(i);
+ final List<String> jobIds = task.get();
+ for (String jobId : jobIds) {
+ assertJobSucceed(jobId);
+ }
+ }
+
+ }
+
+ private void assertJobSucceed(String jobId) {
+ assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
+ }
+
+ private class TestCallable implements Callable<List<String>> {
+
+ private final String methodName;
+ private final CountDownLatch countDownLatch;
+
+ public TestCallable(String methodName, CountDownLatch countDownLatch) {
+ this.methodName = methodName;
+ this.countDownLatch = countDownLatch;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<String> call() throws Exception {
+ try {
+ final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
+ method.setAccessible(true);
+ return (List<String>) method.invoke(BuildIIWithEngineTest.this);
+ } finally {
+ countDownLatch.countDown();
+ }
+ }
+ }
+
+ protected List<String> buildIIInnerJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[0]);
+ }
+
+ protected List<String> buildIILeftJoin() throws Exception {
+ return buildII(TEST_II_INSTANCES[1]);
+ }
+
+ protected List<String> buildII(String iiName) throws Exception {
+ clearSegment(iiName);
+
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+
+ List<String> result = Lists.newArrayList();
+ result.add(buildSegment(iiName, date1, date2));
+ return result;
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance, true);
+ IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
+ IIJob job = iiJobBuilder.buildJob(segment);
+ jobService.addJob(job);
+ waitForJob(job.getId());
+ return job.getId();
+ }
+
+ private int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ public static void main(String[] args) throws Exception {
+ BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
+
+ BuildIIWithEngineTest.beforeClass();
+ instance.before();
+ instance.testBuildII();
+ instance.after();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
new file mode 100644
index 0000000..6bceab7
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -0,0 +1,270 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hive.hcatalog.data.schema.HCatSchema;
+import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.invertedindex.model.IIDesc;
+import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.streaming.StreamMessage;
+import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
+
+/**
+ */
+public class BuildIIWithStreamTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(BuildIIWithStreamTest.class);
+
+ private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
+ private IIManager iiManager;
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+ DeployUtil.overrideJobJarLocations();
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ iiManager = IIManager.getInstance(kylinConfig);
+ iiManager = IIManager.getInstance(kylinConfig);
+ for (String iiInstance : II_NAME) {
+
+ IIInstance ii = iiManager.getII(iiInstance);
+ if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
+ ii.setStatus(RealizationStatusEnum.DISABLED);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ backup();
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = { "--delete", "true" };
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
+ IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
+ JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
+ final String uuid = UUID.randomUUID().toString();
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
+ String insertDataHqls;
+ try {
+ insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
+ } catch (IOException e1) {
+ e1.printStackTrace();
+ throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
+ }
+
+ ShellExecutable step = new ShellExecutable();
+ StringBuffer buf = new StringBuffer();
+ buf.append("hive -e \"");
+ buf.append(dropTableHql + "\n");
+ buf.append(createTableHql + "\n");
+ buf.append(insertDataHqls + "\n");
+ buf.append("\"");
+
+ step.setCmd(buf.toString());
+ logger.info(step.getCmd());
+ step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
+ kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
+ return intermediateTableDesc.getTableName(uuid);
+ }
+
+ private void clearSegment(String iiName) throws Exception {
+ IIInstance ii = iiManager.getII(iiName);
+ ii.getSegments().clear();
+ iiManager.updateII(ii,true);
+ }
+
+ private IISegment createSegment(String iiName) throws Exception {
+ clearSegment(iiName);
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(TimeZone.getTimeZone("GMT"));
+
+ long date1 = 0;
+ long date2 = f.parse("2015-01-01").getTime();
+ return buildSegment(iiName, date1, date2);
+ }
+
+ private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
+ IIInstance iiInstance = iiManager.getII(iiName);
+ IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
+ iiInstance.getSegments().add(segment);
+ iiManager.updateII(iiInstance,true);
+ return segment;
+ }
+
+ private void buildII(String iiName) throws Exception {
+ final IIDesc desc = iiManager.getII(iiName).getDescriptor();
+ final String tableName = createIntermediateTable(desc, kylinConfig);
+ logger.info("intermediate table name:" + tableName);
+ final Configuration conf = new Configuration();
+ HCatInputFormat.setInput(conf, "default", tableName);
+ final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
+ HiveTableReader reader = new HiveTableReader("default", tableName);
+ final List<TblColRef> tblColRefs = desc.listAllColumns();
+ for (TblColRef tblColRef : tblColRefs) {
+ if (desc.isMetricsCol(tblColRef)) {
+ logger.info("matrix:" + tblColRef.getName());
+ } else {
+ logger.info("measure:" + tblColRef.getName());
+ }
+ }
+ LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
+ final IISegment segment = createSegment(iiName);
+ String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
+ ToolRunner.run(new IICreateHTableJob(), args);
+
+ ExecutorService executorService = Executors.newSingleThreadExecutor();
+ final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
+
+ List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
+ int count = sorted.size();
+ for (String[] row : sorted) {
+ logger.info("another row: " + StringUtils.join(row, ","));
+ queue.put(parse(row));
+ }
+
+ reader.close();
+ logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
+ queue.put(StreamMessage.EOF);
+ final Future<?> future = executorService.submit(streamBuilder);
+ try {
+ future.get();
+ } catch (Exception e) {
+ logger.error("stream build failed", e);
+ fail("stream build failed");
+ }
+
+ logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
+ }
+
+ @Test
+ public void test() throws Exception {
+ for (String iiName : II_NAME) {
+ buildII(iiName);
+ IIInstance ii = iiManager.getII(iiName);
+ if (ii.getStatus() != RealizationStatusEnum.READY) {
+ ii.setStatus(RealizationStatusEnum.READY);
+ iiManager.updateII(ii,true);
+ }
+ }
+ }
+
+ private StreamMessage parse(String[] row) {
+ return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
+ }
+
+ private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
+ List<String[]> unsorted = Lists.newArrayList();
+ while (reader.next()) {
+ unsorted.add(reader.getRow());
+ }
+ Collections.sort(unsorted, new Comparator<String[]>() {
+ @Override
+ public int compare(String[] o1, String[] o2) {
+ long t1 = DateFormat.stringToMillis(o1[tsCol]);
+ long t2 = DateFormat.stringToMillis(o2[tsCol]);
+ return Long.compare(t1, t2);
+ }
+ });
+ return unsorted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
deleted file mode 100644
index e99bd01..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class ITBuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(ITBuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
deleted file mode 100644
index 8a26397..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class ITBuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
deleted file mode 100644
index 23f97d0..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class ITBuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(ITBuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = ITBuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(ITBuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- ITBuildIIWithEngineTest instance = new ITBuildIIWithEngineTest();
-
- ITBuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/29058abc/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
deleted file mode 100644
index cfb7bdb..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITBuildIIWithStreamTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.invertedindex.model.IIJoinedFlatTableDesc;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.streaming.StreamMessage;
-import org.apache.kylin.streaming.invertedindex.IIStreamBuilder;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
-import static org.junit.Assert.fail;
-
-/**
- */
-public class ITBuildIIWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITBuildIIWithStreamTest.class);
-
- private static final String[] II_NAME = new String[] { "test_kylin_ii_left_join", "test_kylin_ii_inner_join" };
- private IIManager iiManager;
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- iiManager = IIManager.getInstance(kylinConfig);
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : II_NAME) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
- backup();
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- private String createIntermediateTable(IIDesc desc, KylinConfig kylinConfig) throws IOException {
- IIJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(desc);
- JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
- final String uuid = UUID.randomUUID().toString();
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, uuid);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid, uuid);
- String insertDataHqls;
- try {
- insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, uuid, jobEngineConfig);
- } catch (IOException e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to generate insert data SQL for intermediate table.");
- }
-
- ShellExecutable step = new ShellExecutable();
- StringBuffer buf = new StringBuffer();
- buf.append("hive -e \"");
- buf.append(dropTableHql + "\n");
- buf.append(createTableHql + "\n");
- buf.append(insertDataHqls + "\n");
- buf.append("\"");
-
- step.setCmd(buf.toString());
- logger.info(step.getCmd());
- step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
- kylinConfig.getCliCommandExecutor().execute(step.getCmd(), null);
- return intermediateTableDesc.getTableName(uuid);
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private IISegment createSegment(String iiName) throws Exception {
- clearSegment(iiName);
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
- return buildSegment(iiName, date1, date2);
- }
-
- private IISegment buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance,true);
- return segment;
- }
-
- private void buildII(String iiName) throws Exception {
- final IIDesc desc = iiManager.getII(iiName).getDescriptor();
- final String tableName = createIntermediateTable(desc, kylinConfig);
- logger.info("intermediate table name:" + tableName);
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- final List<TblColRef> tblColRefs = desc.listAllColumns();
- for (TblColRef tblColRef : tblColRefs) {
- if (desc.isMetricsCol(tblColRef)) {
- logger.info("matrix:" + tblColRef.getName());
- } else {
- logger.info("measure:" + tblColRef.getName());
- }
- }
- LinkedBlockingDeque<StreamMessage> queue = new LinkedBlockingDeque<StreamMessage>();
- final IISegment segment = createSegment(iiName);
- String[] args = new String[] { "-iiname", iiName, "-htablename", segment.getStorageLocationIdentifier() };
- ToolRunner.run(new IICreateHTableJob(), args);
-
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- final IIStreamBuilder streamBuilder = new IIStreamBuilder(queue, iiName, segment.getStorageLocationIdentifier(), segment.getIIDesc(), 0);
-
- List<String[]> sorted = getSortedRows(reader, desc.getTimestampColumn());
- int count = sorted.size();
- for (String[] row : sorted) {
- logger.info("another row: " + StringUtils.join(row, ","));
- queue.put(parse(row));
- }
-
- reader.close();
- logger.info("total record count:" + count + " htable:" + segment.getStorageLocationIdentifier());
- queue.put(StreamMessage.EOF);
- final Future<?> future = executorService.submit(streamBuilder);
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- fail("stream build failed");
- }
-
- logger.info("stream build finished, htable name:" + segment.getStorageLocationIdentifier());
- }
-
- @Test
- public void test() throws Exception {
- for (String iiName : II_NAME) {
- buildII(iiName);
- IIInstance ii = iiManager.getII(iiName);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii,true);
- }
- }
- }
-
- private StreamMessage parse(String[] row) {
- return new StreamMessage(System.currentTimeMillis(), StringUtils.join(row, ",").getBytes());
- }
-
- private List<String[]> getSortedRows(HiveTableReader reader, final int tsCol) throws IOException {
- List<String[]> unsorted = Lists.newArrayList();
- while (reader.next()) {
- unsorted.add(reader.getRow());
- }
- Collections.sort(unsorted, new Comparator<String[]>() {
- @Override
- public int compare(String[] o1, String[] o2) {
- long t1 = DateFormat.stringToMillis(o1[tsCol]);
- long t2 = DateFormat.stringToMillis(o2[tsCol]);
- return Long.compare(t1, t2);
- }
- });
- return unsorted;
- }
-
-}
[05/32] incubator-kylin git commit: fix ci: clean broadcase module
Posted by ma...@apache.org.
fix ci: clean broadcase module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9c3715fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9c3715fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9c3715fe
Branch: refs/heads/0.8.0
Commit: 9c3715fed7706016a0b15147fcf927eab5f35e43
Parents: ded2415
Author: honma <ho...@ebay.com>
Authored: Fri May 15 17:32:18 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:11:52 2015 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/Broadcaster.java | 11 +-
.../common/restclient/MultiValueCache.java | 88 ----------
.../common/restclient/SingleValueCache.java | 9 +-
.../org/apache/kylin/cube/CubeDescManager.java | 25 ++-
.../java/org/apache/kylin/cube/CubeManager.java | 161 +++++++++----------
.../apache/kylin/cube/CubeManagerCacheTest.java | 11 +-
.../kylin/invertedindex/IIDescManager.java | 8 +-
.../apache/kylin/invertedindex/IIManager.java | 101 ++++++------
.../invertedindex/IIDescManagerTest.java | 2 +-
.../kylin/job/cube/MergeDictionaryStep.java | 15 +-
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 9 +-
.../job/hadoop/invertedindex/IIBulkLoadJob.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 19 ++-
.../apache/kylin/job/BuildIIWithStreamTest.java | 31 ++--
.../java/org/apache/kylin/job/DeployUtil.java | 27 ++--
.../job/hadoop/cube/MergeCuboidMapperTest.java | 31 ++--
.../apache/kylin/metadata/MetadataManager.java | 33 ++--
.../kylin/metadata/project/ProjectManager.java | 77 ++++-----
.../kylin/rest/controller/CubeController.java | 60 +++----
.../apache/kylin/rest/service/BasicService.java | 15 +-
.../apache/kylin/rest/service/CacheService.java | 132 +++++++--------
.../apache/kylin/rest/service/CubeService.java | 55 +++----
.../apache/kylin/rest/service/JobService.java | 82 +++++-----
.../kylin/rest/service/ProjectService.java | 23 +--
.../kylin/rest/service/CacheServiceTest.java | 51 +++---
.../kylin/rest/service/CubeServiceTest.java | 14 +-
27 files changed, 465 insertions(+), 629 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
index 6fbe413..f745410 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
@@ -35,9 +35,6 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* Broadcast kylin event out
- *
- * @author jianliu
- *
*/
public class Broadcaster {
@@ -113,11 +110,11 @@ public class Broadcaster {
return counter.getAndSet(0);
}
- public static enum EVENT {
+ public enum EVENT {
CREATE("create"), UPDATE("update"), DROP("drop");
private String text;
- private EVENT(String text) {
+ EVENT(String text) {
this.text = text;
}
@@ -136,11 +133,11 @@ public class Broadcaster {
}
}
- public static enum TYPE {
+ public enum TYPE {
ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model"), HYBRID("hybrid");
private String text;
- private TYPE(String text) {
+ TYPE(String text) {
this.text = text;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
deleted file mode 100644
index b479969..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 9848201..fb44206 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -43,8 +43,10 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- final V result = innerCache.put(key, value);
- if (result == null) {
+ //enforce all cache changes coming from REST
+ //final V result = innerCache.put(key, value);
+
+ if (!innerCache.containsKey(key)) {
syncRemote(key, Broadcaster.EVENT.CREATE);
} else {
syncRemote(key, Broadcaster.EVENT.UPDATE);
@@ -57,7 +59,8 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void remove(K key) {
if (innerCache.containsKey(key)) {
- innerCache.remove(key);
+ //enforce all cache changes coming from REST
+ //innerCache.remove(key);
syncRemote(key, Broadcaster.EVENT.DROP);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index ac5fc2b..d406da1 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -18,26 +18,25 @@
package org.apache.kylin.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.Broadcaster;
+import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
+import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Manager class for CubeDesc; extracted from #CubeManager
@@ -108,7 +107,7 @@ public class CubeDescManager {
* @param name
* @throws IOException
*/
- public CubeDesc reloadCubeDesc(String name) throws IOException {
+ public CubeDesc reloadCubeDescLocal(String name) throws IOException {
// Save Source
String path = CubeDesc.getCubeDescResourcePath(name);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 76a6c6d..b986c36 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -52,7 +52,6 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-
/**
* @author yangli9
*/
@@ -138,7 +137,7 @@ public class CubeManager implements IRealizationProvider {
Iterator<CubeInstance> it = list.iterator();
while (it.hasNext()) {
CubeInstance ci = it.next();
- if (descName.equals(ci.getDescName())) {
+ if (descName.equalsIgnoreCase(ci.getDescName())) {
result.add(ci);
}
}
@@ -154,7 +153,7 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return dictInfo;
}
@@ -192,7 +191,7 @@ public class CubeManager implements IRealizationProvider {
cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return snapshot;
}
@@ -202,25 +201,26 @@ public class CubeManager implements IRealizationProvider {
logger.info("Dropping cube '" + cubeName + "'");
// load projects before remove cube from project
- ResourceStore store = getStore();
-
// delete cube instance and cube desc
CubeInstance cube = getCube(cubeName);
- if (deleteDesc && cube.getDescriptor() != null)
- store.deleteResource(cube.getDescriptor().getResourcePath());
-
- store.deleteResource(cube.getResourcePath());
+ if (deleteDesc && cube.getDescriptor() != null) {
+ CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+ }
+ removeCube(cube);
// delete cube from project
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
- // clean cube cache
- this.afterCubeDropped(cube);
-
return cube;
}
+ private void removeCube(CubeInstance cube) throws IOException {
+ // remove cube and update cache
+ getStore().deleteResource(cube.getResourcePath());
+ cubeMap.remove(cube.getName());
+ }
+
// sync on update
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
@@ -228,16 +228,27 @@ public class CubeManager implements IRealizationProvider {
// save cube resource
CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
cube.setOwner(owner);
- saveResource(cube);
+ updateCube(cube, false);
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
return cube;
}
- public CubeInstance updateCube(CubeInstance cube) throws IOException {
+ /**
+ * if not sure whether to enable updateProject, just use it
+ */
+ public CubeInstance updateCube(CubeInstance cube, boolean updateProject) throws IOException {
+
logger.info("Updating cube instance '" + cube.getName());
- saveResource(cube);
+ getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
+ cubeMap.put(cube.getName(), cube);
+
+ if (updateProject) {
+ logger.info("Updating project instance for cube:'" + cube.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.CUBE, cube.getName());
+ }
+
return cube;
}
@@ -258,7 +269,7 @@ public class CubeManager implements IRealizationProvider {
cube.getSegments().add(appendSegment);
cube.getSegments().add(mergeSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
}
@@ -277,19 +288,18 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
-
public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
checkNoBuildingSegment(cube);
CubeSegment newSegment = newSegment(cube, startDate, endDate);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -304,7 +314,7 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -371,28 +381,17 @@ public class CubeManager implements IRealizationProvider {
*
* @param cubeName
*/
- public void loadCubeCache(String cubeName) {
+ public void reloadCubeLocal(String cubeName) {
try {
- loadCubeInstance(CubeInstance.concatResourcePath(cubeName));
+ reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- /**
- * After cube deletion, remove cube related cache
- *
- * @param cube
- */
- public void removeCubeCache(CubeInstance cube) {
- final String cubeName = cube.getName().toUpperCase();
- cubeMap.remove(cubeName);
- usedStorageLocation.removeAll(cubeName);
- }
-
- public void removeCubeCacheLocal(String cubeName) {
+ public void removeCubeLocal(String cubeName) {
cubeMap.removeLocal(cubeName);
- usedStorageLocation.removeAll(cubeName);
+ usedStorageLocation.removeAll(cubeName.toUpperCase());
}
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
@@ -412,18 +411,41 @@ public class CubeManager implements IRealizationProvider {
}
}
- private void saveResource(CubeInstance cube) throws IOException {
- ResourceStore store = getStore();
- store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- this.afterCubeUpdated(cube);
- }
+ private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
+ if (startDate >= endDate)
+ throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
- private void afterCubeUpdated(CubeInstance updatedCube) {
- cubeMap.put(updatedCube.getName(), updatedCube);
+ CubeSegment segment = new CubeSegment();
+ String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
+ segment.setUuid(UUID.randomUUID().toString());
+ segment.setName(incrementalSegName);
+ Date creatTime = new Date();
+ segment.setCreateTimeUTC(creatTime.getTime());
+ segment.setDateRangeStart(startDate);
+ segment.setDateRangeEnd(endDate);
+ segment.setStatus(SegmentStatusEnum.NEW);
+ segment.setStorageLocationIdentifier(generateStorageLocation());
+
+ segment.setCubeInstance(cubeInstance);
+
+ segment.validate();
+ return segment;
}
- private void afterCubeDropped(CubeInstance droppedCube) {
- removeCubeCache(droppedCube);
+ private String generateStorageLocation() {
+ String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
+ String tableName = "";
+ do {
+ StringBuffer sb = new StringBuffer();
+ sb.append(namePrefix);
+ Random ran = new Random();
+ for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
+ sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
+ }
+ tableName = sb.toString();
+ } while (this.usedStorageLocation.containsValue(tableName));
+
+ return tableName;
}
public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
@@ -480,43 +502,6 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
List<CubeSegment> tobe = calculateToBeSegments(cube);
@@ -542,7 +527,7 @@ public class CubeManager implements IRealizationProvider {
cube.setStatus(RealizationStatusEnum.READY);
logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- saveResource(cube);
+ updateCube(cube, true);
}
private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
@@ -579,7 +564,7 @@ public class CubeManager implements IRealizationProvider {
}
firstSeg.validate();
- for (int i = 0, j = 1; j < tobe.size(); ) {
+ for (int i = 0, j = 1; j < tobe.size();) {
CubeSegment is = tobe.get(i);
CubeSegment js = tobe.get(j);
js.validate();
@@ -644,16 +629,16 @@ public class CubeManager implements IRealizationProvider {
logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
for (String path : paths) {
- loadCubeInstance(path);
+ reloadCubeLocalAt(path);
}
logger.debug("Loaded " + paths.size() + " Cube(s)");
}
- private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
+ private synchronized CubeInstance reloadCubeLocalAt(String path) throws IOException {
ResourceStore store = getStore();
- CubeInstance cubeInstance = null;
+ CubeInstance cubeInstance;
try {
cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
cubeInstance.setConfig(config);
@@ -668,7 +653,7 @@ public class CubeManager implements IRealizationProvider {
cubeMap.putLocal(cubeName, cubeInstance);
for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
+ usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier());
}
return cubeInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 5e64e01..7c74993 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -18,18 +18,17 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.*;
-
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
+import static org.junit.Assert.assertEquals;
/**
* @author yangli9
@@ -68,7 +67,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
- cubeManager.updateCube(createdCube);
+ cubeManager.updateCube(createdCube,true);
assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
index ae38955..cf0af86 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
@@ -18,6 +18,7 @@
package org.apache.kylin.invertedindex;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -27,7 +28,6 @@ import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,7 +90,7 @@ public class IIDescManager {
reloadAllIIDesc();
}
- public List<IIDesc> listAllDesc(){
+ public List<IIDesc> listAllDesc() {
return new ArrayList<IIDesc>(iiDescMap.values());
}
@@ -105,7 +105,7 @@ public class IIDescManager {
* @param name
* @throws IOException
*/
- public IIDesc reloadIIDesc(String name) throws IOException {
+ public IIDesc reloadIIDescLocal(String name) throws IOException {
// Save Source
String path = IIDesc.getIIDescResourcePath(name);
@@ -170,7 +170,7 @@ public class IIDescManager {
}
public void removeIIDescLocal(String name) throws IOException {
- iiDescMap.remove(name);
+ iiDescMap.removeLocal(name);
}
private void reloadAllIIDesc() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
index c1a0765..ea40ebd 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
@@ -18,6 +18,8 @@
package org.apache.kylin.invertedindex;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -31,6 +33,8 @@ import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.IRealizationProvider;
@@ -39,7 +43,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -92,7 +99,7 @@ public class IIManager implements IRealizationProvider {
private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX);
// for generation hbase table name of a new segment
- private HashSet<String> usedStorageLocation = new HashSet<String>();
+ private Multimap<String, String> usedStorageLocation = HashMultimap.create();
private IIManager(KylinConfig config) throws IOException {
logger.info("Initializing IIManager with config " + config);
@@ -106,7 +113,6 @@ public class IIManager implements IRealizationProvider {
}
public IIInstance getII(String iiName) {
- iiName = iiName.toUpperCase();
return iiMap.get(iiName);
}
@@ -137,7 +143,7 @@ public class IIManager implements IRealizationProvider {
DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
iiSeg.putDictResPath(column, dict.getResourcePath());
}
- saveResource(iiSeg.getIIInstance());
+ updateII(iiSeg.getIIInstance(), false);
}
/**
@@ -168,58 +174,55 @@ public class IIManager implements IRealizationProvider {
if (this.getII(ii.getName()) != null)
throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
- // other logic is the same as update.
- return updateII(ii);
- }
-
- public void updateIIStreamingOffset(String iiName, int partition, long offset) throws IOException {
-
- }
-
-
- public IIInstance updateII(IIInstance ii) throws IOException {
- logger.info("Updating II instance '" + ii.getName());
-
- // save resource
- saveResource(ii);
-
- logger.info("II with " + ii.getSegments().size() + " segments is saved");
+ this.updateII(ii, false);
+ String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName();
+ ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner());
return ii;
}
- public void loadIICache(String iiName) {
+ public void reloadIILocal(String iiName) {
try {
- loadIIInstance(IIInstance.concatResourcePath(iiName));
+ reloadIILocalAt(IIInstance.concatResourcePath(iiName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- public void removeIICache(IIInstance ii) {
- iiMap.remove(ii.getName());
+ public IIInstance dropII(String iiName, boolean deleteDesc) throws IOException {
+ logger.info("Dropping II '" + iiName + "'");
+
+ IIInstance ii = getII(iiName);
- for (IISegment segment : ii.getSegments()) {
- usedStorageLocation.remove(segment.getName());
+ if (deleteDesc && ii.getDescriptor() != null) {
+ IIDescManager.getInstance(config).removeIIDesc(ii.getDescriptor());
}
+
+ removeII(ii);
+ ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.INVERTED_INDEX, iiName);
+
+ return ii;
}
- public void removeIILocalCache(String name) {
- iiMap.removeLocal(name);
- //TODO
- // for (IISegment segment : ii.getSegments()) {
- // usedStorageLocation.remove(segment.getName());
- // }
+ private void removeII(IIInstance ii) throws IOException {
+ getStore().deleteResource(ii.getResourcePath());
+ iiMap.remove(ii.getName());
}
- private void saveResource(IIInstance ii) throws IOException {
- ResourceStore store = getStore();
- store.putResource(ii.getResourcePath(), ii, II_SERIALIZER);
- this.afterIIUpdated(ii);
+ public void removeIILocal(String name) {
+ iiMap.removeLocal(name);
+ usedStorageLocation.removeAll(name.toUpperCase());
}
- private void afterIIUpdated(IIInstance updatedII) {
- iiMap.put(updatedII.getName(), updatedII);
+ public void updateII(IIInstance ii, boolean updateProject) throws IOException {
+ logger.info("Updating II instance : " + ii.getName());
+ getStore().putResource(ii.getResourcePath(), ii, II_SERIALIZER);
+ iiMap.put(ii.getName(), ii);
+
+ if (updateProject) {
+ logger.info("Updating project instance for ii: " + ii.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.INVERTED_INDEX, ii.getName());
+ }
}
/**
@@ -251,7 +254,7 @@ public class IIManager implements IRealizationProvider {
int idx = (int) (Math.random() * ALPHA_NUM.length());
sb.append(ALPHA_NUM.charAt(idx));
}
- if (usedStorageLocation.contains(sb.toString())) {
+ if (usedStorageLocation.containsValue(sb.toString())) {
continue;
} else {
return sb.toString();
@@ -266,31 +269,31 @@ public class IIManager implements IRealizationProvider {
logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));
for (String path : paths) {
- loadIIInstance(path);
+ reloadIILocalAt(path);
}
logger.debug("Loaded " + paths.size() + " II(s)");
}
- private synchronized IIInstance loadIIInstance(String path) throws IOException {
+ private synchronized IIInstance reloadIILocalAt(String path) throws IOException {
ResourceStore store = getStore();
logger.debug("Loading IIInstance " + store.getReadableResourcePath(path));
- IIInstance IIInstance = null;
+ IIInstance ii = null;
try {
- IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER);
- IIInstance.setConfig(config);
+ ii = store.getResource(path, IIInstance.class, II_SERIALIZER);
+ ii.setConfig(config);
- if (StringUtils.isBlank(IIInstance.getName()))
+ if (StringUtils.isBlank(ii.getName()))
throw new IllegalStateException("IIInstance name must not be blank");
- iiMap.putLocal(IIInstance.getName(), IIInstance);
+ iiMap.putLocal(ii.getName(), ii);
- for (IISegment segment : IIInstance.getSegments()) {
- usedStorageLocation.add(segment.getName());
+ for (IISegment segment : ii.getSegments()) {
+ usedStorageLocation.put(ii.getName().toUpperCase(), segment.getStorageLocationIdentifier());
}
- return IIInstance;
+ return ii;
} catch (Exception e) {
logger.error("Error during load ii instance " + path, e);
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
index a13a0d1..a456dcd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
@@ -95,7 +95,7 @@ public class IIDescManagerTest extends LocalFileMetadataTestCase {
Assert.assertEquals(desc, newDesc);
// reload the cache
- mgr.reloadIIDesc(TEST_II_DESC_NAME);
+ mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
index b9aa7a4..d0a7db3 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
@@ -18,16 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -42,6 +34,9 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.model.TblColRef;
+import java.io.IOException;
+import java.util.*;
+
public class MergeDictionaryStep extends AbstractExecutable {
private static final String CUBE_NAME = "cubeName";
@@ -68,7 +63,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
- mgr.updateCube(cube);
+ mgr.updateCube(cube,false);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index e9c2a2c..8140003 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -18,11 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -33,6 +30,8 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
+import java.io.IOException;
+
/**
*/
public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
@@ -128,7 +127,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
if (segmentReady) {
cubeManager.promoteNewlyBuiltSegments(cube, segment);
} else {
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube, true);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
index f636af4..145a3c8 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
@@ -64,7 +64,7 @@ public class IIBulkLoadJob extends AbstractHadoopJob {
IISegment seg = ii.getFirstSegment();
seg.setStorageLocationIdentifier(tableName);
seg.setStatus(SegmentStatusEnum.READY);
- mgr.updateII(ii);
+ mgr.updateII(ii,true);
return hbaseExitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 812eb9d..547a6e7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -263,7 +263,7 @@ public class BuildCubeWithEngineTest {
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index af8682e..9c0219a 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -61,7 +61,7 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
- protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -104,7 +104,7 @@ public class BuildIIWithEngineTest {
}
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof IIJob){
+ if (jobService.getJob(jobId) instanceof IIJob) {
jobService.deleteJob(jobId);
}
}
@@ -115,7 +115,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
}
@@ -127,7 +127,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
backup();
@@ -137,7 +137,7 @@ public class BuildIIWithEngineTest {
@Ignore
public void testBuildII() throws Exception {
- String[] testCase = new String[]{"buildIIInnerJoin", "buildIILeftJoin"};
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
@@ -183,10 +183,9 @@ public class BuildIIWithEngineTest {
}
protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
+ return buildII(TEST_II_INSTANCES[0]);
}
-
protected List<String> buildIILeftJoin() throws Exception {
return buildII(TEST_II_INSTANCES[1]);
}
@@ -208,14 +207,14 @@ public class BuildIIWithEngineTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance, true);
IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
IIJob job = iiJobBuilder.buildJob(segment);
jobService.addJob(job);
@@ -224,7 +223,7 @@ public class BuildIIWithEngineTest {
}
private int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
+ String[] args = { "--delete", "true" };
int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
return exitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 65faad7..6bceab7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -34,17 +34,7 @@
package org.apache.kylin.job;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
@@ -77,7 +67,16 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
/**
*/
@@ -109,7 +108,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
@@ -169,7 +168,7 @@ public class BuildIIWithStreamTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private IISegment createSegment(String iiName) throws Exception {
@@ -186,7 +185,7 @@ public class BuildIIWithStreamTest {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance,true);
return segment;
}
@@ -243,7 +242,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiName);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 329eef1..6d348bd 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -18,35 +18,30 @@
package org.apache.kylin.job;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.maven.model.Model;
-import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
-import org.apache.tools.ant.filters.StringInputStream;
-import org.codehaus.plexus.util.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HiveClient;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
+import org.apache.tools.ant.filters.StringInputStream;
+import org.codehaus.plexus.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
public class DeployUtil {
@SuppressWarnings("unused")
@@ -65,7 +60,7 @@ public class DeployUtil {
// update cube desc signature.
for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeManager.getInstance(config()).updateCube(cube);
+ CubeManager.getInstance(config()).updateCube(cube,true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 1d0b16b..9f7e5f9 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -18,34 +18,29 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
+import org.apache.kylin.dict.*;
import org.apache.kylin.dict.lookup.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertTrue;
@@ -139,7 +134,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
// cubeManager.saveResource(segment.getCubeInstance());
// cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
isFirstSegment = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 331bd83..2833218 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -18,23 +18,6 @@
package org.apache.kylin.metadata;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
@@ -45,8 +28,22 @@ import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import javax.xml.crypto.Data;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Serves (and caches) metadata for Kylin instance.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 029179f..76b1c0f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -102,20 +102,19 @@ public class ProjectManager {
logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT));
for (String path : paths) {
- reloadProjectAt(path);
+ reloadProjectLocalAt(path);
}
wireProjectAndRealizations(projectMap.values());
logger.debug("Loaded " + projectMap.size() + " Project(s)");
}
- public ProjectInstance reloadProject(String project) throws IOException {
- return reloadProjectAt(ProjectInstance.concatResourcePath(project));
+ public ProjectInstance reloadProjectLocal(String project) throws IOException {
+ return reloadProjectLocalAt(ProjectInstance.concatResourcePath(project));
}
- private ProjectInstance reloadProjectAt(String path) throws IOException {
- ResourceStore store = getStore();
+ private ProjectInstance reloadProjectLocalAt(String path) throws IOException {
- ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
+ ProjectInstance projectInstance = getStore().getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
if (projectInstance == null) {
logger.warn("reload project at path:" + path + " not found, this:" + this.toString());
return null;
@@ -159,16 +158,16 @@ public class ProjectManager {
}
public ProjectInstance createProject(String projectName, String owner, String description) throws IOException {
- logger.info("Creating project '" + projectName);
+ logger.info("Creating project " + projectName);
ProjectInstance currentProject = getProject(projectName);
if (currentProject == null) {
- currentProject = ProjectInstance.create(projectName, owner, description, null,null);
+ currentProject = ProjectInstance.create(projectName, owner, description, null, null);
} else {
throw new IllegalStateException("The project named " + projectName + "already exists");
}
- saveResource(currentProject);
+ updateProject(currentProject);
return currentProject;
}
@@ -189,21 +188,30 @@ public class ProjectManager {
logger.info("Dropping project '" + projectInstance.getName() + "'");
- deleteResource(projectInstance);
+ removeProject(projectInstance);
return projectInstance;
}
+ //passive update due to underlying realization update
+ public void updateProject(RealizationType type, String realizationName) throws IOException {
+ for (ProjectInstance proj : findProjects(type, realizationName)) {
+ updateProject(proj);
+ }
+ }
+
+ //update project itself
public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException {
if (!project.getName().equals(newName)) {
ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc);
- // FIXME table lost??
+
newProject.setCreateTimeUTC(project.getCreateTimeUTC());
newProject.recordUpdateTime(System.currentTimeMillis());
newProject.setRealizationEntries(project.getRealizationEntries());
+ newProject.setTables(project.getTables());
- deleteResource(project);
- saveResource(newProject);
+ removeProject(project);
+ updateProject(newProject);
return newProject;
} else {
@@ -213,12 +221,24 @@ public class ProjectManager {
if (project.getUuid() == null)
project.updateRandomUuid();
- saveResource(project);
+ updateProject(project);
return project;
}
}
+ private void updateProject(ProjectInstance prj) throws IOException {
+ getStore().putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
+ projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
+ clearL2Cache();
+ }
+
+ private void removeProject(ProjectInstance proj) throws IOException {
+ getStore().deleteResource(proj.getResourcePath());
+ projectMap.remove(norm(proj.getName()));
+ clearL2Cache();
+ }
+
public boolean isModelInProject(String projectName, String modelName) {
return this.getProject(projectName).containsModel(modelName);
}
@@ -231,7 +251,7 @@ public class ProjectManager {
public void removeModelFromProjects(String modelName) throws IOException {
for (ProjectInstance projectInstance : findProjects(modelName)) {
projectInstance.removeModel(modelName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -239,10 +259,10 @@ public class ProjectManager {
String newProjectName = ProjectInstance.getNormalizedProjectName(project);
ProjectInstance newProject = getProject(newProjectName);
if (newProject == null) {
- throw new IllegalArgumentException("Project "+newProjectName+" does not exist.");
+ throw new IllegalArgumentException("Project " + newProjectName + " does not exist.");
}
newProject.addModel(modelName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -252,7 +272,6 @@ public class ProjectManager {
return addRealizationToProject(type, realizationName, newProjectName, owner);
}
-
private ProjectInstance addRealizationToProject(RealizationType type, String realizationName, String project, String user) throws IOException {
String newProjectName = norm(project);
ProjectInstance newProject = getProject(newProjectName);
@@ -260,7 +279,7 @@ public class ProjectManager {
newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")");
}
newProject.addRealizationEntry(type, realizationName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -268,7 +287,7 @@ public class ProjectManager {
public void removeRealizationsFromProjects(RealizationType type, String realizationName) throws IOException {
for (ProjectInstance projectInstance : findProjects(type, realizationName)) {
projectInstance.removeRealization(type, realizationName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -283,26 +302,10 @@ public class ProjectManager {
projectInstance.addTable(table.getIdentity());
}
- saveResource(projectInstance);
+ updateProject(projectInstance);
return projectInstance;
}
- private void saveResource(ProjectInstance prj) throws IOException {
- ResourceStore store = getStore();
- store.putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
-
- prj = reloadProjectAt(prj.getResourcePath());
- projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
- clearL2Cache();
- }
-
- private void deleteResource(ProjectInstance proj) throws IOException {
- ResourceStore store = getStore();
- store.deleteResource(proj.getResourcePath());
- projectMap.remove(norm(proj.getName()));
- clearL2Cache();
- }
-
public List<ProjectInstance> findProjects(RealizationType type, String realizationName) {
List<ProjectInstance> result = Lists.newArrayList();
for (ProjectInstance prj : projectMap.values()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b1f9b4a..466a007 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -20,25 +20,18 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -58,12 +51,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import com.codahale.metrics.annotation.Metered;
import com.fasterxml.jackson.core.JsonParseException;
@@ -86,10 +74,10 @@ public class CubeController extends BasicController {
@Autowired
private JobService jobService;
- @RequestMapping(value = "", method = {RequestMethod.GET})
+ @RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "listCubes")
- public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName,@RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value="limit",required = false) Integer limit, @RequestParam(value = "offset" ,required = false) Integer offset) {
+ public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
}
@@ -101,7 +89,7 @@ public class CubeController extends BasicController {
* @throws UnknownHostException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -123,7 +111,7 @@ public class CubeController extends BasicController {
* @param notifyList
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/notify_list", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
@ResponseBody
public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -141,7 +129,7 @@ public class CubeController extends BasicController {
}
- @RequestMapping(value = "/{cubeName}/cost", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCubeCost")
public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
@@ -154,7 +142,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
@ResponseBody
public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
try {
@@ -172,7 +160,7 @@ public class CubeController extends BasicController {
*
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
try {
@@ -191,7 +179,7 @@ public class CubeController extends BasicController {
* @throws SchedulerException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest jobBuildRequest) {
try {
@@ -208,7 +196,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/disable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "disableCube")
public CubeInstance disableCube(@PathVariable String cubeName) {
@@ -227,7 +215,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/purge", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "purgeCube")
public CubeInstance purgeCube(@PathVariable String cubeName) {
@@ -246,7 +234,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/enable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "enableCube")
public CubeInstance enableCube(@PathVariable String cubeName) {
@@ -264,7 +252,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}", method = {RequestMethod.DELETE})
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
@ResponseBody
@Metered(name = "deleteCube")
public void deleteCube(@PathVariable String cubeName) {
@@ -287,7 +275,7 @@ public class CubeController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.POST})
+ @RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
@Metered(name = "saveCube")
public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
@@ -324,7 +312,7 @@ public class CubeController extends BasicController {
* @throws JsonProcessingException
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.PUT})
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCube")
public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
@@ -338,11 +326,17 @@ public class CubeController extends BasicController {
// Check if the cube is editable
if (!cubeService.isCubeDescEditable(desc)) {
- String error = "Cube desc " + desc.getName().toUpperCase() + " is not editable.";
+ String error = "Purge the related cube before editing its desc. Desc name: " + desc.getName();
updateRequest(cubeRequest, false, error);
return cubeRequest;
}
+ //cube renaming:
+ if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) {
+ deleteCube(cubeRequest.getCubeName());
+ saveCubeDesc(cubeRequest);
+ }
+
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
@@ -373,7 +367,7 @@ public class CubeController extends BasicController {
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "getHBaseInfo")
public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
@@ -429,14 +423,12 @@ public class CubeController extends BasicController {
return desc;
}
-
-
/**
* @return
*/
private String omitMessage(List<String> errors) {
StringBuffer buffer = new StringBuffer();
- for (Iterator<String> iterator = errors.iterator(); iterator.hasNext(); ) {
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
String string = (String) iterator.next();
buffer.append(string);
buffer.append("\n");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index c0f366d..0ae9376 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -23,15 +23,15 @@ import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -41,7 +41,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
@@ -49,7 +48,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
-
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -78,6 +76,7 @@ public abstract class BasicService {
}
public void removeOLAPDataSource(String project) {
+ logger.info("removeOLAPDataSource is called for project " + project);
if (StringUtils.isEmpty(project))
throw new IllegalArgumentException("removeOLAPDataSource: project name not given");
@@ -85,9 +84,9 @@ public abstract class BasicService {
olapDataSources.remove(project);
}
- public static void resetOLAPDataSources() {
+ public static void removeAllOLAPDataSources() {
// brutal, yet simplest way
- logger.info("resetOLAPDataSources is called.");
+ logger.info("removeAllOLAPDataSources is called.");
olapDataSources.clear();
}
@@ -135,7 +134,7 @@ public abstract class BasicService {
public void cleanDataCache() {
CubeManager.clearCache();
ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
+ removeAllOLAPDataSources();
}
public final KylinConfig getKylinConfig() {
@@ -184,7 +183,7 @@ public abstract class BasicService {
if (cubeName == null) {
return true;
}
- return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
+ return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
} else {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 6ea563f..9b89932 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -24,16 +24,13 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
import org.apache.kylin.job.cube.CubingJobBuilder;
import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.QueryController;
import org.slf4j.Logger;
@@ -44,7 +41,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.util.List;
/**
*/
@@ -61,58 +57,48 @@ public class CacheService extends BasicService {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().loadCubeCache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
- mergeCubeOnNewSegmentReady(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().reloadCubeDesc(cacheKey);
- break;
- case PROJECT:
- getProjectManager().reloadProject(cacheKey);
- break;
- case INVERTED_INDEX:
- getIIManager().loadIICache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDesc(cacheKey);
- break;
- case TABLE:
- getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case DATA_MODEL:
- getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case ALL:
- getMetadataManager().reload();
- CubeDescManager.clearCache();
- CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
- break;
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().reloadCubeLocal(cacheKey);
+ mergeCubeOnNewSegmentReady(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().reloadCubeDescLocal(cacheKey);
+ break;
+ case PROJECT:
+ ProjectInstance projectInstance = getProjectManager().reloadProjectLocal(cacheKey);
+ removeOLAPDataSource(projectInstance.getName());
+ break;
+ case INVERTED_INDEX:
+ getIIManager().reloadIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().reloadIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ getMetadataManager().reloadTableCache(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case DATA_MODEL:
+ getMetadataManager().reloadDataModelDesc(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case ALL:
+ getMetadataManager().reload();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ IIDescManager.clearCache();
+ IIManager.clearCache();
+ ProjectManager.clearCache();
+ removeAllOLAPDataSources();
+ break;
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
}
-
- }
-
- private void cleanProjectCacheByRealization(RealizationType type, String realizationName) throws IOException {
- List<ProjectInstance> projectInstances = getProjectManager().findProjects(type, realizationName);
- for (ProjectInstance pi : projectInstances) {
- getProjectManager().reloadProject(pi.getName());
- removeOLAPDataSource(pi.getName());
- }
}
@Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
@@ -120,27 +106,27 @@ public class CacheService extends BasicService {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().removeCubeCacheLocal(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().removeLocalCubeDesc(cacheKey);
- break;
- case PROJECT:
- ProjectManager.clearCache();
- break;
- case INVERTED_INDEX:
- getIIManager().removeIILocalCache(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
- case TABLE:
- throw new UnsupportedOperationException(log);
- case DATA_MODEL:
- throw new UnsupportedOperationException(log);
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().removeCubeLocal(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().removeLocalCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ ProjectManager.clearCache();
+ break;
+ case INVERTED_INDEX:
+ getIIManager().removeIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().removeIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ throw new UnsupportedOperationException(log);
+ case DATA_MODEL:
+ throw new UnsupportedOperationException(log);
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
[08/32] incubator-kylin git commit: KYLIN-697 non-integration tests
all passed
Posted by ma...@apache.org.
KYLIN-697 non-integration tests all passed
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4b6e6e2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4b6e6e2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4b6e6e2e
Branch: refs/heads/0.8.0
Commit: 4b6e6e2e41699a85e3528533ab1a0088e724a21b
Parents: 04eca53
Author: honma <ho...@ebay.com>
Authored: Mon May 25 11:13:46 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:17:15 2015 +0800
----------------------------------------------------------------------
.../persistence/HBaseResourceStoreTest.java | 208 --------------
.../persistence/ITHBaseResourceStoreTest.java | 208 ++++++++++++++
.../kylin/common/util/ITMockHTableTest.java | 158 +++++++++++
.../kylin/common/util/MockHTableTest.java | 158 -----------
.../apache/kylin/dict/HiveTableReaderTest.java | 51 ----
.../kylin/dict/ITHiveTableReaderTest.java | 51 ++++
.../kylin/dict/ITSnapshotManagerTest.java | 82 ++++++
.../apache/kylin/dict/SnapshotManagerTest.java | 82 ------
.../Installation/Off Hadoop CLI Installation.md | 2 +-
.../MISC/FAQ on Kylin Installation and Usage.md | 2 +-
...n Kylin test case with HBase Mini Cluster.md | 2 +-
jdbc/kylin_jdbc.log.2014-12-22 | 18 ++
.../kylin/job/BuildCubeWithEngineTest.java | 279 -------------------
.../kylin/job/BuildCubeWithStreamTest.java | 224 ---------------
.../apache/kylin/job/BuildIIWithEngineTest.java | 254 -----------------
.../apache/kylin/job/BuildIIWithStreamTest.java | 270 ------------------
.../apache/kylin/job/IIStreamBuilderTest.java | 108 -------
.../kylin/job/ITBuildCubeWithEngineTest.java | 279 +++++++++++++++++++
.../kylin/job/ITBuildCubeWithStreamTest.java | 224 +++++++++++++++
.../kylin/job/ITBuildIIWithEngineTest.java | 254 +++++++++++++++++
.../kylin/job/ITBuildIIWithStreamTest.java | 270 ++++++++++++++++++
.../apache/kylin/job/ITIIStreamBuilderTest.java | 108 +++++++
.../impl/threadpool/DefaultSchedulerTest.java | 102 -------
.../impl/threadpool/ITDefaultSchedulerTest.java | 97 +++++++
.../tool/HiveSourceTableLoaderTest.java | 59 ----
.../tool/ITHiveSourceTableLoaderTest.java | 59 ++++
pom.xml | 24 +-
.../kylin/query/test/CombinationTest.java | 72 -----
.../apache/kylin/query/test/IIQueryTest.java | 85 ------
.../kylin/query/test/ITCombinationTest.java | 72 +++++
.../apache/kylin/query/test/ITIIQueryTest.java | 85 ++++++
.../kylin/query/test/ITKylinQueryTest.java | 234 ++++++++++++++++
.../apache/kylin/query/test/KylinQueryTest.java | 234 ----------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 245 ++++++++++++++++
.../org/apache/kylin/jdbc/JDBCDriverTest.java | 245 ----------------
.../rest/controller/ITTableControllerTest.java | 74 +++++
.../rest/controller/TableControllerTest.java | 74 -----
.../kylin/rest/service/CacheServiceTest.java | 11 +-
.../storage/hbase/ITInvertedIndexHBaseTest.java | 114 ++++++++
.../storage/hbase/InvertedIndexHBaseTest.java | 114 --------
.../kylin/storage/test/ITStorageTest.java | 154 ++++++++++
.../apache/kylin/storage/test/StorageTest.java | 154 ----------
.../kylin/streaming/ITKafkaConsumerTest.java | 108 +++++++
.../kylin/streaming/ITKafkaRequesterTest.java | 76 +++++
.../kylin/streaming/KafkaConsumerTest.java | 108 -------
.../kylin/streaming/KafkaRequesterTest.java | 76 -----
46 files changed, 2979 insertions(+), 2989 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
deleted file mode 100644
index f4788f4..0000000
--- a/common/src/test/java/org/apache/kylin/common/persistence/HBaseResourceStoreTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.persistence;
-
-import static org.junit.Assert.*;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class HBaseResourceStoreTest extends HBaseMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testHBaseStore() throws Exception {
- testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
- }
-
- @Test
- public void testHBaseStoreWithLargeCell() throws Exception {
- String path = "/cube/_test_large_cell.json";
- String largeContent = "THIS_IS_A_LARGE_CELL";
- StringEntity content = new StringEntity(largeContent);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int origSize = config.getHBaseKeyValueSize();
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- try {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
-
- store.deleteResource(path);
-
- store.putResource(path, content, StringEntity.serializer);
- assertTrue(store.exists(path));
- StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
- assertEquals(content, t);
-
- Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
- assertTrue(fileSystem.exists(redirectPath));
-
- FSDataInputStream in = fileSystem.open(redirectPath);
- assertEquals(largeContent, in.readUTF());
- in.close();
-
- store.deleteResource(path);
- } finally {
- config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
- store.deleteResource(path);
- }
- }
-
- void testAStore(ResourceStore store) throws IOException {
- String dir1 = "/cube";
- String path1 = "/cube/_test.json";
- StringEntity content1 = new StringEntity("anything");
- String dir2 = "/table";
- String path2 = "/table/_test.json";
- StringEntity content2 = new StringEntity("something");
-
- // cleanup legacy if any
- store.deleteResource(path1);
- store.deleteResource(path2);
-
- StringEntity t;
-
- // put/get
- store.putResource(path1, content1, StringEntity.serializer);
- assertTrue(store.exists(path1));
- t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
- assertEquals(content1, t);
-
- store.putResource(path2, content2, StringEntity.serializer);
- assertTrue(store.exists(path2));
- t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
- assertEquals(content2, t);
-
- // overwrite
- t.str = "new string";
- store.putResource(path2, t, StringEntity.serializer);
-
- // write conflict
- try {
- t.setLastModified(t.lastModified - 1);
- store.putResource(path2, t, StringEntity.serializer);
- fail("write conflict should trigger IllegalStateException");
- } catch (IllegalStateException e) {
- // expected
- }
-
- // list
- ArrayList<String> list;
-
- list = store.listResources(dir1);
- assertTrue(list.contains(path1));
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(dir2);
- assertTrue(list.contains(path2));
- assertTrue(list.contains(path1) == false);
-
- list = store.listResources("/");
- assertTrue(list.contains(dir1));
- assertTrue(list.contains(dir2));
- assertTrue(list.contains(path1) == false);
- assertTrue(list.contains(path2) == false);
-
- list = store.listResources(path1);
- assertNull(list);
- list = store.listResources(path2);
- assertNull(list);
-
- // delete/exist
- store.deleteResource(path1);
- assertTrue(store.exists(path1) == false);
- list = store.listResources(dir1);
- assertTrue(list == null || list.contains(path1) == false);
-
- store.deleteResource(path2);
- assertTrue(store.exists(path2) == false);
- list = store.listResources(dir2);
- assertTrue(list == null || list.contains(path2) == false);
- }
-
- public static class StringEntity extends RootPersistentEntity {
-
- static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
- @Override
- public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
- out.writeUTF(obj.str);
- }
-
- @Override
- public StringEntity deserialize(DataInputStream in) throws IOException {
- String str = in.readUTF();
- return new StringEntity(str);
- }
- };
-
- String str;
-
- public StringEntity(String str) {
- this.str = str;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = super.hashCode();
- result = prime * result + ((str == null) ? 0 : str.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this)
- return true;
- if (!(obj instanceof StringEntity))
- return false;
- return StringUtils.equals(this.str, ((StringEntity) obj).str);
- }
-
- @Override
- public String toString() {
- return str;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
new file mode 100644
index 0000000..6195423
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/persistence/ITHBaseResourceStoreTest.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+
+public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase {
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHBaseStore() throws Exception {
+ testAStore(ResourceStore.getStore(KylinConfig.getInstanceFromEnv()));
+ }
+
+ @Test
+ public void testHBaseStoreWithLargeCell() throws Exception {
+ String path = "/cube/_test_large_cell.json";
+ String largeContent = "THIS_IS_A_LARGE_CELL";
+ StringEntity content = new StringEntity(largeContent);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int origSize = config.getHBaseKeyValueSize();
+ ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+
+ try {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", String.valueOf(largeContent.length() - 1));
+
+ store.deleteResource(path);
+
+ store.putResource(path, content, StringEntity.serializer);
+ assertTrue(store.exists(path));
+ StringEntity t = store.getResource(path, StringEntity.class, StringEntity.serializer);
+ assertEquals(content, t);
+
+ Path redirectPath = ((HBaseResourceStore) store).bigCellHDFSPath(path);
+ Configuration hconf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fileSystem = FileSystem.get(hconf);
+ assertTrue(fileSystem.exists(redirectPath));
+
+ FSDataInputStream in = fileSystem.open(redirectPath);
+ assertEquals(largeContent, in.readUTF());
+ in.close();
+
+ store.deleteResource(path);
+ } finally {
+ config.setProperty("kylin.hbase.client.keyvalue.maxsize", "" + origSize);
+ store.deleteResource(path);
+ }
+ }
+
+ void testAStore(ResourceStore store) throws IOException {
+ String dir1 = "/cube";
+ String path1 = "/cube/_test.json";
+ StringEntity content1 = new StringEntity("anything");
+ String dir2 = "/table";
+ String path2 = "/table/_test.json";
+ StringEntity content2 = new StringEntity("something");
+
+ // cleanup legacy if any
+ store.deleteResource(path1);
+ store.deleteResource(path2);
+
+ StringEntity t;
+
+ // put/get
+ store.putResource(path1, content1, StringEntity.serializer);
+ assertTrue(store.exists(path1));
+ t = store.getResource(path1, StringEntity.class, StringEntity.serializer);
+ assertEquals(content1, t);
+
+ store.putResource(path2, content2, StringEntity.serializer);
+ assertTrue(store.exists(path2));
+ t = store.getResource(path2, StringEntity.class, StringEntity.serializer);
+ assertEquals(content2, t);
+
+ // overwrite
+ t.str = "new string";
+ store.putResource(path2, t, StringEntity.serializer);
+
+ // write conflict
+ try {
+ t.setLastModified(t.lastModified - 1);
+ store.putResource(path2, t, StringEntity.serializer);
+ fail("write conflict should trigger IllegalStateException");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+
+ // list
+ ArrayList<String> list;
+
+ list = store.listResources(dir1);
+ assertTrue(list.contains(path1));
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(dir2);
+ assertTrue(list.contains(path2));
+ assertTrue(list.contains(path1) == false);
+
+ list = store.listResources("/");
+ assertTrue(list.contains(dir1));
+ assertTrue(list.contains(dir2));
+ assertTrue(list.contains(path1) == false);
+ assertTrue(list.contains(path2) == false);
+
+ list = store.listResources(path1);
+ assertNull(list);
+ list = store.listResources(path2);
+ assertNull(list);
+
+ // delete/exist
+ store.deleteResource(path1);
+ assertTrue(store.exists(path1) == false);
+ list = store.listResources(dir1);
+ assertTrue(list == null || list.contains(path1) == false);
+
+ store.deleteResource(path2);
+ assertTrue(store.exists(path2) == false);
+ list = store.listResources(dir2);
+ assertTrue(list == null || list.contains(path2) == false);
+ }
+
+ public static class StringEntity extends RootPersistentEntity {
+
+ static final Serializer<StringEntity> serializer = new Serializer<StringEntity>() {
+ @Override
+ public void serialize(StringEntity obj, DataOutputStream out) throws IOException {
+ out.writeUTF(obj.str);
+ }
+
+ @Override
+ public StringEntity deserialize(DataInputStream in) throws IOException {
+ String str = in.readUTF();
+ return new StringEntity(str);
+ }
+ };
+
+ String str;
+
+ public StringEntity(String str) {
+ this.str = str;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((str == null) ? 0 : str.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+ if (!(obj instanceof StringEntity))
+ return false;
+ return StringUtils.equals(this.str, ((StringEntity) obj).str);
+ }
+
+ @Override
+ public String toString() {
+ return str;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
new file mode 100644
index 0000000..0913ad7
--- /dev/null
+++ b/common/src/test/java/org/apache/kylin/common/util/ITMockHTableTest.java
@@ -0,0 +1,158 @@
+package org.apache.kylin.common.util;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.HBaseConnection;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * MockHTable Test.
+ *
+ * compare to real HTable.
+ */
+
+public class ITMockHTableTest extends HBaseMetadataTestCase {
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private static HTableInterface mock;
+ private static HTableInterface real;
+ private static String emailCF = "EMAIL";
+ private static String addressCF = "ADDRESS";
+ private static String emailCQ = "email";
+ private static String addressCQ = "address";
+ private static String tableName = "MockHTable-TEST";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+
+ staticCreateTestMetadata();
+
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
+ real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
+ mock = new MockHTable(tableName, emailCF, addressCF);
+
+ setupDefaultData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
+ staticCleanupTestMetadata();
+ }
+
+ private static void setupDefaultData() throws IOException, InterruptedException {
+ Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
+ Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
+ Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
+ mutateAll(john, jane, me);
+ }
+
+ private static Put createPutForPerson(String name, String email, String address) {
+ Put person = new Put(name.getBytes());
+ person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
+ person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
+ return person;
+ }
+
+ private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
+ real.batch(Lists.newArrayList(rows));
+ mock.batch(Lists.newArrayList(rows));
+ }
+
+ @Test
+ public void test_get() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(false));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_get_with_filter() throws Exception {
+ Get get = new Get("John Doe".getBytes());
+ get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
+
+ ));
+
+ Result realResult = real.get(get);
+ Result mockResult = mock.get(get);
+
+ assertThat(realResult.isEmpty(), is(true));
+ assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
+ }
+
+ @Test
+ public void test_exists() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_include_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(true));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_exists_with_only_not_exist_column() throws IOException {
+ Get get = new Get("John Doe".getBytes());
+ get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
+ boolean realResult = real.exists(get);
+ boolean mockResult = mock.exists(get);
+
+ assertThat(realResult, is(false));
+ assertThat(realResult, is(mockResult));
+ }
+
+ @Test
+ public void test_scan_with_filter() throws Exception {
+ Scan scan = new Scan();
+ scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ logger.debug("mock : {}", mockResult);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+
+ @Test
+ public void test_scan_for_pre_match() throws Exception {
+ Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
+
+ ResultScanner realResult = real.getScanner(scan);
+ ResultScanner mockResult = mock.getScanner(scan);
+
+ assertThat(Iterables.size(realResult), is(2));
+ assertThat(Iterables.size(mockResult), is(2));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java b/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
deleted file mode 100644
index 825fafc..0000000
--- a/common/src/test/java/org/apache/kylin/common/util/MockHTableTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-package org.apache.kylin.common.util;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
-
-/**
- * MockHTable Test.
- *
- * compare to real HTable.
- */
-
-public class MockHTableTest extends HBaseMetadataTestCase {
- private final Logger logger = LoggerFactory.getLogger(this.getClass());
-
- private static HTableInterface mock;
- private static HTableInterface real;
- private static String emailCF = "EMAIL";
- private static String addressCF = "ADDRESS";
- private static String emailCQ = "email";
- private static String addressCQ = "address";
- private static String tableName = "MockHTable-TEST";
-
- @BeforeClass
- public static void setUp() throws Exception {
-
- staticCreateTestMetadata();
-
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName, emailCF, addressCF);
- real = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(tableName);
- mock = new MockHTable(tableName, emailCF, addressCF);
-
- setupDefaultData();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- HBaseConnection.deleteTable(KylinConfig.getInstanceFromEnv().getStorageUrl(), tableName);
- staticCleanupTestMetadata();
- }
-
- private static void setupDefaultData() throws IOException, InterruptedException {
- Put john = createPutForPerson("John Doe", "john.doe@email.com", "US");
- Put jane = createPutForPerson("Jane Doe", "jane.doe@other.email.com", "US");
- Put me = createPutForPerson("Yunsang Choi", "oddpoet@gmail", "jeju");
- mutateAll(john, jane, me);
- }
-
- private static Put createPutForPerson(String name, String email, String address) {
- Put person = new Put(name.getBytes());
- person.add(emailCF.getBytes(), emailCQ.getBytes(), email.getBytes());
- person.add(addressCF.getBytes(), addressCQ.getBytes(), address.getBytes());
- return person;
- }
-
- private static <R extends Row> void mutateAll(R... rows) throws IOException, InterruptedException, IOException {
- real.batch(Lists.newArrayList(rows));
- mock.batch(Lists.newArrayList(rows));
- }
-
- @Test
- public void test_get() throws Exception {
- Get get = new Get("John Doe".getBytes());
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(false));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_get_with_filter() throws Exception {
- Get get = new Get("John Doe".getBytes());
- get.setFilter(new SingleColumnValueFilter(emailCF.getBytes(), emailCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "WRONG EMAIL".getBytes()
-
- ));
-
- Result realResult = real.get(get);
- Result mockResult = mock.get(get);
-
- assertThat(realResult.isEmpty(), is(true));
- assertThat(mockResult.isEmpty(), is(realResult.isEmpty()));
- }
-
- @Test
- public void test_exists() throws IOException {
- Get get = new Get("John Doe".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_include_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), emailCQ.getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(true));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_exists_with_only_not_exist_column() throws IOException {
- Get get = new Get("John Doe".getBytes());
- get.addColumn(emailCF.getBytes(), "NOT_EXIST_COLUMN".getBytes());
- boolean realResult = real.exists(get);
- boolean mockResult = mock.exists(get);
-
- assertThat(realResult, is(false));
- assertThat(realResult, is(mockResult));
- }
-
- @Test
- public void test_scan_with_filter() throws Exception {
- Scan scan = new Scan();
- scan.setFilter(new SingleColumnValueFilter(addressCF.getBytes(), addressCQ.getBytes(), CompareFilter.CompareOp.EQUAL, "US".getBytes()));
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- logger.debug("mock : {}", mockResult);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-
- @Test
- public void test_scan_for_pre_match() throws Exception {
- Scan scan = new Scan("J".getBytes(), "K".getBytes()); // start with 'J' only
-
- ResultScanner realResult = real.getScanner(scan);
- ResultScanner mockResult = mock.getScanner(scan);
-
- assertThat(Iterables.size(realResult), is(2));
- assertThat(Iterables.size(mockResult), is(2));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
deleted file mode 100644
index 25279c6..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/HiveTableReaderTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case need the hive runtime; Please run it with sandbox;
- * @author shaoshi
- *
- * It is in the exclude list of default profile in pom.xml
- */
-public class HiveTableReaderTest extends HBaseMetadataTestCase {
-
-
- @Test
- public void test() throws IOException {
- HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
- int rowNumber = 0;
- while (reader.next()) {
- String[] row = reader.getRow();
- Assert.assertEquals(9, row.length);
- //System.out.println(ArrayUtils.toString(row));
- rowNumber++;
- }
-
- reader.close();
- Assert.assertEquals(10000, rowNumber);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
new file mode 100644
index 0000000..8559f8b
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITHiveTableReaderTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTableReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * This test case need the hive runtime; Please run it with sandbox;
+ * @author shaoshi
+ *
+ * It is in the exclude list of default profile in pom.xml
+ */
+public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
+
+
+ @Test
+ public void test() throws IOException {
+ HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
+ int rowNumber = 0;
+ while (reader.next()) {
+ String[] row = reader.getRow();
+ Assert.assertEquals(9, row.length);
+ //System.out.println(ArrayUtils.toString(row));
+ rowNumber++;
+ }
+
+ reader.close();
+ Assert.assertEquals(10000, rowNumber);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
new file mode 100644
index 0000000..e59fcda
--- /dev/null
+++ b/dictionary/src/test/java/org/apache/kylin/dict/ITSnapshotManagerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.dict.lookup.HiveTable;
+import org.apache.kylin.dict.lookup.SnapshotManager;
+import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.dict.lookup.TableReader;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author yangli9
+ *
+ */
+public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
+
+ SnapshotManager snapshotMgr;
+
+ @Before
+ public void setup() throws Exception {
+ createTestMetadata();
+ snapshotMgr = SnapshotManager.getInstance(getTestConfig());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanupTestMetadata();
+ }
+
+ @Test
+ public void basicTest() throws Exception {
+ String tableName = "EDW.TEST_SITES";
+ HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
+ TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
+ String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
+
+ snapshotMgr.wipeoutCache();
+
+ SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
+
+ // compare hive & snapshot
+ TableReader hiveReader = hiveTable.getReader();
+ TableReader snapshotReader = snapshot.getReader();
+
+ while (true) {
+ boolean hiveNext = hiveReader.next();
+ boolean snapshotNext = snapshotReader.next();
+ assertEquals(hiveNext, snapshotNext);
+
+ if (hiveNext == false)
+ break;
+
+ String[] hiveRow = hiveReader.getRow();
+ String[] snapshotRow = snapshotReader.getRow();
+ assertArrayEquals(hiveRow, snapshotRow);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java b/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
deleted file mode 100644
index 0ae8e95..0000000
--- a/dictionary/src/test/java/org/apache/kylin/dict/SnapshotManagerTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import static org.junit.Assert.*;
-
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.dict.lookup.TableReader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-
-/**
- * @author yangli9
- *
- */
-public class SnapshotManagerTest extends HBaseMetadataTestCase {
-
- SnapshotManager snapshotMgr;
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- snapshotMgr = SnapshotManager.getInstance(getTestConfig());
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void basicTest() throws Exception {
- String tableName = "EDW.TEST_SITES";
- HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
-
- snapshotMgr.wipeoutCache();
-
- SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);
-
- // compare hive & snapshot
- TableReader hiveReader = hiveTable.getReader();
- TableReader snapshotReader = snapshot.getReader();
-
- while (true) {
- boolean hiveNext = hiveReader.next();
- boolean snapshotNext = snapshotReader.next();
- assertEquals(hiveNext, snapshotNext);
-
- if (hiveNext == false)
- break;
-
- String[] hiveRow = hiveReader.getRow();
- String[] snapshotRow = snapshotReader.getRow();
- assertArrayEquals(hiveRow, snapshotRow);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/docs/Installation/Off Hadoop CLI Installation.md
----------------------------------------------------------------------
diff --git a/docs/Installation/Off Hadoop CLI Installation.md b/docs/Installation/Off Hadoop CLI Installation.md
index f83ed5d..a1877e3 100644
--- a/docs/Installation/Off Hadoop CLI Installation.md
+++ b/docs/Installation/Off Hadoop CLI Installation.md
@@ -53,7 +53,7 @@ An alternative to the host replacement is updating your `hosts` file to resolve
Run a end-to-end cube building test
- mvn test -Dtest=org.apache.kylin.job.BuildCubeWithEngineTest -DfailIfNoTests=false
+ mvn test -Dtest=org.apache.kylin.job.ITBuildCubeWithEngineTest -DfailIfNoTests=false
Run other tests, the end-to-end cube building test is exclueded
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/docs/MISC/FAQ on Kylin Installation and Usage.md
----------------------------------------------------------------------
diff --git a/docs/MISC/FAQ on Kylin Installation and Usage.md b/docs/MISC/FAQ on Kylin Installation and Usage.md
index a568614..6f24512 100644
--- a/docs/MISC/FAQ on Kylin Installation and Usage.md
+++ b/docs/MISC/FAQ on Kylin Installation and Usage.md
@@ -27,7 +27,7 @@ My Cluster is running on Cloudera Distribution CDH 5.2.0.
#### Unable to load a big cube as HTable, with java.lang.OutOfMemoryError: unable to create new native thread
HBase (as of writing) allocates one thread per region when bulk loading a HTable. Try reduce the number of regions of your cube by setting its "capacity" to "MEDIUM" or "LARGE". Also tweaks OS & JVM can allow more threads, for example see [this article](http://blog.egilh.com/2006/06/2811aspx.html).
-#### Failed to run BuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
+#### Failed to run ITBuildCubeWithEngineTest, saying failed to connect to hbase while hbase is active
User may get this error when first time run hbase client, please check the error trace to see whether there is an error saying couldn't access a folder like "/hadoop/hbase/local/jars"; If that folder doesn't exist, create it.
#### SUM(field) returns a negtive result while all the numbers in this field are > 0
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/docs/Test/Run Kylin test case with HBase Mini Cluster.md
----------------------------------------------------------------------
diff --git a/docs/Test/Run Kylin test case with HBase Mini Cluster.md b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
index 6088eb9..107da81 100644
--- a/docs/Test/Run Kylin test case with HBase Mini Cluster.md
+++ b/docs/Test/Run Kylin test case with HBase Mini Cluster.md
@@ -8,7 +8,7 @@ mvn test -P sandbox
* Startup a HBase minicluster and update KylinConfig with the dynamic HBase configurations;
* Create Kylin metadata tables and import six example cube tables;
-* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running BuildCubeWithEngineTest）
+* Import the hbase data from a tar ball from local: examples/test_case_data/minicluster/hbase-export.tar.gz (the hbase-export.tar.gz will be updated on complete of running ITBuildCubeWithEngineTest）
* After all test cases be completed, shutdown minicluster and cleanup KylinConfig cache;
### To ensure Mini cluster can run successfully, you need:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/jdbc/kylin_jdbc.log.2014-12-22
----------------------------------------------------------------------
diff --git a/jdbc/kylin_jdbc.log.2014-12-22 b/jdbc/kylin_jdbc.log.2014-12-22
new file mode 100644
index 0000000..7c5e001
--- /dev/null
+++ b/jdbc/kylin_jdbc.log.2014-12-22
@@ -0,0 +1,18 @@
+[main]:[2014-12-22 11:47:05,477][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,486][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 11:47:05,487][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 11:47:05,534][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,077][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,086][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@3add750e]
+[main]:[2014-12-22 15:57:49,087][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 15:57:49,134][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 15:57:49,135][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,081][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,090][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl.getTables(KylinMetaImpl.java:108)] - Get tables with conn KylinConnectionImpl [baseUrl=test_url, project=test_db, metaProject=com.kylinolap.jdbc.KylinMetaImpl$MetaProject@21300853]
+[main]:[2014-12-22 17:13:45,091][DEBUG][com.kylinolap.jdbc.KylinMetaImpl$MetaProject.getMetaTables(KylinMetaImpl.java:489)] - getMetaTables with catalog:null, schema:null, table:null
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.KylinConnectionImpl.<init>(KylinConnectionImpl.java:68)] - Kylin base url test_url, project name test_db
+[main]:[2014-12-22 17:13:45,141][DEBUG][com.kylinolap.jdbc.Driver$1.onConnectionInit(Driver.java:111)] - Connection inited.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
deleted file mode 100644
index 547a6e7..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.cube.CubingJobBuilder;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-public class BuildCubeWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
-
- private CubeManager cubeManager;
-
- private DefaultScheduler scheduler;
-
- protected ExecutableManager jobService;
-
- private static final Log logger = LogFactory.getLog(BuildCubeWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- cubeManager = CubeManager.getInstance(kylinConfig);
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof CubingJob){
- jobService.deleteJob(jobId);
- }
- }
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- DeployUtil.prepareTestData("left", "test_kylin_cube_with_slr_left_join_empty");
- testInner();
- testLeft();
- }
-
- private void testInner() throws Exception {
- String[] testCase = new String[]{
- "testInnerJoinCube",
- "testInnerJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void testLeft() throws Exception {
- String[] testCase = new String[]{
- "testLeftJoinCube",
- "testLeftJoinCube2",
- };
- runTestAndAssertSucceed(testCase);
- }
-
- private void runTestAndAssertSucceed(String[] testCase) throws Exception {
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- try {
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
- } catch (Exception ex) {
- logger.error(ex);
- throw ex;
- }
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals("The job '" + jobId + "' is failed.", ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildCubeWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube2() throws Exception {
- clearSegment("test_kylin_cube_with_slr_empty");
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testInnerJoinCube() throws Exception {
- clearSegment("test_kylin_cube_without_slr_empty");
-
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- // this cube's start date is 0, end date is 20501112000000
- long date1 = 0;
- long date2 = f.parse("2013-01-01").getTime();
-
-
- // this cube doesn't support incremental build, always do full build
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_without_slr_empty", date1, date2));
- return result;
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube2() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- // this cube's start date is 0, end date is 20120601000000
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2012-06-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
-
- // then submit an append job, start date is 20120601000000, end
- // date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
- dateEnd = f.parse("2022-01-01").getTime();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- @SuppressWarnings("unused") // called by reflection
- private List<String> testLeftJoinCube() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- long dateEnd = f.parse("2050-11-12").getTime();
-
- // this cube's start date is 0, end date is 20501112000000
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(cubeName, dateStart, dateEnd));
- return result;
-
- }
-
- private void clearSegment(String cubeName) throws Exception {
- CubeInstance cube = cubeManager.getCube(cubeName);
- cube.getSegments().clear();
- cubeManager.updateCube(cube,true);
- }
-
-
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
- CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
- CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
- CubingJob job = cubingJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
deleted file mode 100644
index dc42dc5..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hive.hcatalog.data.schema.HCatSchema;
-import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.lookup.HiveTableReader;
-import org.apache.kylin.job.hadoop.cubev2.IGTRecordWriter;
-import org.apache.kylin.job.hadoop.cubev2.InMemCubeBuilder;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.gridtable.GTRecord;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- *
- * This class is going to be deleted
- */
-@Ignore("For dev testing")
-public class BuildCubeWithStreamTest {
-
- private static final Logger logger = LoggerFactory.getLogger(BuildCubeWithStreamTest.class);
-
- private KylinConfig kylinConfig;
- private CubeManager cubeManager;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
- DeployUtil.overrideJobJarLocations();
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cubeManager = CubeManager.getInstance(kylinConfig);
-
- }
-
- @After
- public void after() {
- HBaseMetadataTestCase.staticCleanupTestMetadata();
- }
-
- @Test
- public void test() throws Exception {
- CubeInstance cube = cubeManager.getCube("test_kylin_cube_without_slr_left_join_empty");
- final CubeDesc desc = cube.getDescriptor();
- // cube.getSegments().clear();
- // cubeManager.updateCube(cube);
-
- CubeSegment cubeSegment = cube.getSegment("19700101000000_20150401000000", SegmentStatusEnum.NEW);
- Map<TblColRef, Dictionary<?>> dictionaryMap = Maps.newHashMap();
-
-//
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = cubeSegment.getDictionary(col);
- if (dict == null) {
- throw new IllegalArgumentException("Dictionary for " + col + " was not found.");
- }
- logger.info("Dictionary for " + col + " was put into dictionary map.");
- dictionaryMap.put(col, cubeSegment.getDictionary(col));
- }
- }
- }
-
-// final String tableName = createIntermediateTable(desc, kylinConfig, null);
- String tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a24dec89_efbd_425f_9a5f_8b78dd1412af"; // has 3089 records;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_desc_19700101000000_20130112000000_a5e1eb5d_da6b_475d_9807_be0b61f03215"; // only 20 rows;
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150302000000_0a183367_f245_43d1_8850_1c138c8514c3";
-// tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150301000000_ce061464_7962_4642_bd7d_7c3d8fbe9389";
- tableName = "kylin_intermediate_test_kylin_cube_without_slr_left_join_desc_19700101000000_20150401000000_fb7ae579_d987_4900_a3b7_c60c731cd269"; // 2 million records
- logger.info("intermediate table name:" + tableName);
-
-
- ArrayBlockingQueue queue = new ArrayBlockingQueue<List<String>>(10000);
-
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(queue, cube, dictionaryMap, new ConsoleGTRecordWriter());
- ExecutorService executorService = Executors.newSingleThreadExecutor();
- Future<?> future = executorService.submit(cubeBuilder);
-
- final Configuration conf = new Configuration();
- HCatInputFormat.setInput(conf, "default", tableName);
- final HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- logger.info(StringUtils.join(tableSchema.getFieldNames(), "\n"));
- HiveTableReader reader = new HiveTableReader("default", tableName);
- List<String> row;
- int counter = 0;
- while (reader.next()) {
- row = reader.getRowAsList();
- queue.put(row);
- counter++;
- if(counter == 200000)
- break;
- }
- queue.put(new ArrayList<String>(0));
- reader.close();
-
- try {
- future.get();
- } catch (Exception e) {
- logger.error("stream build failed", e);
- throw new IOException("Failed to build cube ", e);
- }
-
- logger.info("stream build finished");
- }
-
-
- private void buildDictionary(List<List<String>> table, CubeDesc desc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
- SetMultimap<TblColRef, String> valueMap = HashMultimap.create();
-
- List<TblColRef> dimColumns = desc.listDimensionColumnsExcludingDerived();
- for (List<String> row : table) {
- for (int i = 0; i < dimColumns.size(); i++) {
- String cell = row.get(i);
- valueMap.put(dimColumns.get(i), cell);
- }
- }
-
- for (DimensionDesc dim : desc.getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (desc.getRowkey().isUseDictionary(col)) {
- Dictionary dict = DictionaryGenerator.buildDictionaryFromValueList(col.getType(), Collections2.transform(valueMap.get(col), new Function<String, byte[]>() {
- @Nullable
- @Override
- public byte[] apply(String input) {
- if (input == null)
- return null;
- return input.getBytes();
- }
- }));
-
- logger.info("Building dictionary for " + col);
- dictionaryMap.put(col, dict);
- }
- }
- }
-
- }
-
-
- class ConsoleGTRecordWriter implements IGTRecordWriter {
-
- boolean verbose = false;
-
- @Override
- public void write(Long cuboidId, GTRecord record) throws IOException {
- if (verbose)
- System.out.println(record.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4b6e6e2e/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
deleted file mode 100644
index 9c0219a..0000000
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.invertedindex.IIJob;
-import org.apache.kylin.job.invertedindex.IIJobBuilder;
-import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.junit.*;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-import java.util.concurrent.*;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author shaoshi
- */
-public class BuildIIWithEngineTest {
-
- private JobEngineConfig jobEngineConfig;
- private IIManager iiManager;
-
- private DefaultScheduler scheduler;
- protected ExecutableManager jobService;
-
- protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
-
- private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
-
- protected void waitForJob(String jobId) {
- while (true) {
- AbstractExecutable job = jobService.getJob(jobId);
- if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
- } else {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- //DeployUtil.initCliWorkDir();
- // DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
-
- final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- jobService = ExecutableManager.getInstance(kylinConfig);
- scheduler = DefaultScheduler.getInstance();
- scheduler.init(new JobEngineConfig(kylinConfig));
- if (!scheduler.hasStarted()) {
- throw new RuntimeException("scheduler has not been started");
- }
- jobEngineConfig = new JobEngineConfig(kylinConfig);
- for (String jobId : jobService.getAllJobIds()) {
- if (jobService.getJob(jobId) instanceof IIJob) {
- jobService.deleteJob(jobId);
- }
- }
-
- iiManager = IIManager.getInstance(kylinConfig);
- for (String iiInstance : TEST_II_INSTANCES) {
-
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
- ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii, true);
- }
- }
- }
-
- @After
- public void after() throws Exception {
-
- for (String iiInstance : TEST_II_INSTANCES) {
- IIInstance ii = iiManager.getII(iiInstance);
- if (ii.getStatus() != RealizationStatusEnum.READY) {
- ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii, true);
- }
- }
- backup();
- }
-
- @Test
- @Ignore
- public void testBuildII() throws Exception {
-
- String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
- ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
- final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
- for (int i = 0; i < testCase.length; i++) {
- tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
- }
- countDownLatch.await();
- for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
- }
- }
-
- }
-
- private void assertJobSucceed(String jobId) {
- assertEquals(ExecutableState.SUCCEED, jobService.getOutput(jobId).getState());
- }
-
- private class TestCallable implements Callable<List<String>> {
-
- private final String methodName;
- private final CountDownLatch countDownLatch;
-
- public TestCallable(String methodName, CountDownLatch countDownLatch) {
- this.methodName = methodName;
- this.countDownLatch = countDownLatch;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public List<String> call() throws Exception {
- try {
- final Method method = BuildIIWithEngineTest.class.getDeclaredMethod(methodName);
- method.setAccessible(true);
- return (List<String>) method.invoke(BuildIIWithEngineTest.this);
- } finally {
- countDownLatch.countDown();
- }
- }
- }
-
- protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
- }
-
- protected List<String> buildIILeftJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[1]);
- }
-
- protected List<String> buildII(String iiName) throws Exception {
- clearSegment(iiName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- long date1 = 0;
- long date2 = f.parse("2015-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
- result.add(buildSegment(iiName, date1, date2));
- return result;
- }
-
- private void clearSegment(String iiName) throws Exception {
- IIInstance ii = iiManager.getII(iiName);
- ii.getSegments().clear();
- iiManager.updateII(ii,true);
- }
-
- private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
- IIInstance iiInstance = iiManager.getII(iiName);
- IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
- iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance, true);
- IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
- IIJob job = iiJobBuilder.buildJob(segment);
- jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
- }
-
- private int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- public static void main(String[] args) throws Exception {
- BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
-
- BuildIIWithEngineTest.beforeClass();
- instance.before();
- instance.testBuildII();
- instance.after();
-
- }
-
-}
[31/32] incubator-kylin git commit: KYLIN-697 change HdfsOpsTest to
integration test
Posted by ma...@apache.org.
KYLIN-697 change HdfsOpsTest to integration test
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/77494791
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/77494791
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/77494791
Branch: refs/heads/0.8.0
Commit: 77494791c5f3c2301f01fd8902ec25ad4b844312
Parents: 5b6960e
Author: honma <ho...@ebay.com>
Authored: Tue May 26 21:08:35 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:28 2015 +0800
----------------------------------------------------------------------
.../kylin/job/hadoop/hdfs/HdfsOpsTest.java | 67 --------------------
.../kylin/job/hadoop/hdfs/ITHdfsOpsTest.java | 66 +++++++++++++++++++
2 files changed, 66 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/77494791/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
deleted file mode 100644
index 3b4a6af..0000000
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/HdfsOpsTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.hadoop.hdfs;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-
-/**
- */
-public class HdfsOpsTest extends LocalFileMetadataTestCase {
-
- FileSystem fileSystem;
-
- @Before
- public void setup() throws Exception {
-
- this.createTestMetadata();
-
- Configuration hconf = new Configuration();
-
- fileSystem = FileSystem.get(hconf);
- }
-
- @Test
- public void TestPath() throws IOException {
- String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
- Path coprocessorDir = new Path(hdfsWorkingDirectory, "test");
- fileSystem.mkdirs(coprocessorDir);
-
- Path newFile = new Path(coprocessorDir, "test_file");
- newFile = newFile.makeQualified(fileSystem.getUri(), null);
- FSDataOutputStream stream = fileSystem.create(newFile);
- stream.write(new byte[] { 0, 1, 2 });
- stream.close();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/77494791/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/ITHdfsOpsTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/ITHdfsOpsTest.java
new file mode 100644
index 0000000..8710681
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/hdfs/ITHdfsOpsTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ */
+public class ITHdfsOpsTest extends HBaseMetadataTestCase {
+
+ FileSystem fileSystem;
+
+ @Before
+ public void setup() throws Exception {
+
+ this.createTestMetadata();
+
+ Configuration hconf = new Configuration();
+
+ fileSystem = FileSystem.get(hconf);
+ }
+
+ @Test
+ public void TestPath() throws IOException {
+ String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
+ Path coprocessorDir = new Path(hdfsWorkingDirectory, "test");
+ fileSystem.mkdirs(coprocessorDir);
+
+ Path newFile = new Path(coprocessorDir, "test_file");
+ newFile = newFile.makeQualified(fileSystem.getUri(), null);
+ FSDataOutputStream stream = fileSystem.create(newFile);
+ stream.write(new byte[] { 0, 1, 2 });
+ stream.close();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+}
[04/32] incubator-kylin git commit: fix ci: clean broadcase module
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d168ddd..ba62bb2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -63,7 +63,6 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.net.UnknownHostException;
import java.util.*;
/**
@@ -81,7 +80,7 @@ public class CubeService extends BasicService {
private AccessService accessService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> listAllCubes(final String cubeName, final String projectName,final String modelName) {
+ public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName) {
List<CubeInstance> cubeInstances = null;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -93,14 +92,14 @@ public class CubeService extends BasicService {
List<CubeInstance> filterModelCubes = new ArrayList<CubeInstance>();
- if(modelName!=null){
+ if (modelName != null) {
for (CubeInstance cubeInstance : cubeInstances) {
boolean isCubeMatch = cubeInstance.getDescriptor().getModelName().toLowerCase().equals(modelName.toLowerCase());
if (isCubeMatch) {
filterModelCubes.add(cubeInstance);
}
}
- }else{
+ } else {
filterModelCubes = cubeInstances;
}
@@ -116,12 +115,12 @@ public class CubeService extends BasicService {
return filterCubes;
}
- public List<CubeInstance> getCubes(final String cubeName, final String projectName,final String modelName ,final Integer limit, final Integer offset) {
+ public List<CubeInstance> getCubes(final String cubeName, final String projectName, final String modelName, final Integer limit, final Integer offset) {
List<CubeInstance> cubes;
- cubes = listAllCubes(cubeName, projectName,modelName);
+ cubes = listAllCubes(cubeName, projectName, modelName);
- if(limit==null||offset==null){
+ if (limit == null || offset == null) {
return cubes;
}
@@ -154,7 +153,7 @@ public class CubeService extends BasicService {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
cube.setOwner(owner);
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
}
public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
@@ -174,7 +173,6 @@ public class CubeService extends BasicService {
createdDesc = getCubeDescManager().updateCubeDesc(desc);
}
-
if (!createdDesc.getError().isEmpty()) {
if (isNew) {
getCubeDescManager().removeCubeDesc(createdDesc);
@@ -235,7 +233,7 @@ public class CubeService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
- public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
+ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
@@ -276,20 +274,14 @@ public class CubeService extends BasicService {
accessService.clean(cube, true);
}
- public boolean isCubeEditable(CubeInstance ci) {
- return ci.getStatus() == RealizationStatusEnum.DISABLED;
- }
-
public boolean isCubeDescEditable(CubeDesc cd) {
- List<CubeInstance> list = getCubeManager().getCubesByDesc(cd.getName());
- if (list.isEmpty()) {
+ String cubeName = getCubeNameFromDesc(cd.getName());
+ CubeInstance cube = getCubeManager().getCube(cubeName);
+ if (cube == null) {
return true;
}
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- if (!isCubeEditable(it.next())) {
- return false;
- }
+ if (cube.getSegments().size() != 0) {
+ return false;
}
return true;
}
@@ -306,14 +298,6 @@ public class CubeService extends BasicService {
}
}
- public void reloadCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).loadCubeCache(cubeName);
- }
-
- public void removeCubeCache(String cubeName) {
- CubeManager.getInstance(this.getConfig()).removeCubeCacheLocal(cubeName);
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
@@ -323,7 +307,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -349,7 +333,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -361,7 +345,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.DISABLED);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -398,7 +382,7 @@ public class CubeService extends BasicService {
cube.setStatus(RealizationStatusEnum.READY);
try {
- return getCubeManager().updateCube(cube);
+ return getCubeManager().updateCube(cube, true);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
@@ -511,7 +495,6 @@ public class CubeService extends BasicService {
getExecutableManager().addJob(job);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
CubeDesc desc = cube.getDescriptor();
@@ -543,7 +526,7 @@ public class CubeService extends BasicService {
}
}
cube.getSegments().clear();
- CubeManager.getInstance(getConfig()).updateCube(cube);
+ CubeManager.getInstance(getConfig()).updateCube(cube, true);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -557,7 +540,6 @@ public class CubeService extends BasicService {
getProjectManager().addTableDescToProject(tables, project);
}
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
MetadataManager metaMgr = getMetadataManager();
@@ -569,5 +551,4 @@ public class CubeService extends BasicService {
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index e370a27..7c96801 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -100,20 +100,20 @@ public class JobService extends BasicService {
private ExecutableState parseToExecutableState(JobStatusEnum status) {
switch (status) {
- case DISCARDED:
- return ExecutableState.DISCARDED;
- case ERROR:
- return ExecutableState.ERROR;
- case FINISHED:
- return ExecutableState.SUCCEED;
- case NEW:
- return ExecutableState.READY;
- case PENDING:
- return ExecutableState.READY;
- case RUNNING:
- return ExecutableState.RUNNING;
- default:
- throw new RuntimeException("illegal status:" + status);
+ case DISCARDED:
+ return ExecutableState.DISCARDED;
+ case ERROR:
+ return ExecutableState.ERROR;
+ case FINISHED:
+ return ExecutableState.SUCCEED;
+ case NEW:
+ return ExecutableState.READY;
+ case PENDING:
+ return ExecutableState.READY;
+ case RUNNING:
+ return ExecutableState.RUNNING;
+ default:
+ throw new RuntimeException("illegal status:" + status);
}
}
@@ -213,37 +213,37 @@ public class JobService extends BasicService {
private JobStatusEnum parseToJobStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
private JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- default:
- throw new RuntimeException("invalid state:" + state);
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ default:
+ throw new RuntimeException("invalid state:" + state);
}
}
@@ -264,7 +264,7 @@ public class JobService extends BasicService {
final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
if (segment != null && segment.getStatus() == SegmentStatusEnum.NEW) {
cubeInstance.getSegments().remove(segment);
- getCubeManager().updateCube(cubeInstance);
+ getCubeManager().updateCube(cubeInstance, true);
}
getExecutableManager().discardJob(jobId);
return jobInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 2013a9f..abaecd2 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -18,12 +18,12 @@
package org.apache.kylin.rest.service;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.CreateProjectRequest;
import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.security.AclPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -31,11 +31,9 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.security.AclPermission;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
/**
* @author xduo
@@ -107,12 +105,5 @@ public class ProjectService extends BasicService {
accessService.clean(project, true);
}
- public void reloadProjectCache(String name) throws IOException {
- getProjectManager().reloadProject(name);
- }
-
- public void removeProjectCache(String name) {
- ProjectManager.clearCache();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 7a88ba9..559ec89 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -77,7 +77,13 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
context.addServlet(new ServletHolder(new BroadcasterReceiveServlet(new BroadcasterReceiveServlet.BroadcasterHandler() {
@Override
public void handle(String type, String name, String event) {
- final CacheService cacheService = new CacheService() {
+ final CacheService serviceA = new CacheService() {
+ @Override
+ public KylinConfig getConfig() {
+ return configA;
+ }
+ };
+ final CacheService serviceB = new CacheService() {
@Override
public KylinConfig getConfig() {
return configB;
@@ -87,20 +93,26 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
Broadcaster.EVENT wipeEvent = Broadcaster.EVENT.getEvent(event);
final String log = "wipe cache type: " + wipeType + " event:" + wipeEvent + " name:" + name;
logger.info(log);
- counter.incrementAndGet();
- switch (wipeEvent) {
+ try {
+ switch (wipeEvent) {
case CREATE:
case UPDATE:
- cacheService.rebuildCache(wipeType, name);
+ serviceA.rebuildCache(wipeType, name);
+ serviceB.rebuildCache(wipeType, name);
break;
case DROP:
- cacheService.removeCache(wipeType, name);
+ serviceA.removeCache(wipeType, name);
+ serviceB.removeCache(wipeType, name);
break;
default:
throw new RuntimeException("invalid type:" + wipeEvent);
+ }
+ } finally {
+ counter.incrementAndGet();
}
}
})), "/");
+
server.start();
}
@@ -146,12 +158,15 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
private static CubeManager getCubeManager(KylinConfig config) throws Exception {
return CubeManager.getInstance(config);
}
+
private static ProjectManager getProjectManager(KylinConfig config) throws Exception {
return ProjectManager.getInstance(config);
}
+
private static CubeDescManager getCubeDescManager(KylinConfig config) throws Exception {
return CubeDescManager.getInstance(config);
}
+
private static MetadataManager getMetadataManager(KylinConfig config) throws Exception {
return MetadataManager.getInstance(config);
}
@@ -200,10 +215,11 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
- assertNotNull(cubeManager.getCube(cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertNotNull(cubeManager.getCube(cubeName));
assertNotNull(cubeManagerB.getCube(cubeName));
assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
@@ -215,24 +231,24 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
CubeSegment segment = new CubeSegment();
segment.setName("test_segment");
cube.getSegments().add(segment);
- cubeManager.updateCube(cube);
- //only one for update cube
- assertEquals(1, broadcaster.getCounterAndClear());
- waitForCounterAndClear(1);
+ cubeManager.updateCube(cube, true);
+ //one for cube update, one for project update
+ assertEquals(2, broadcaster.getCounterAndClear());
+ waitForCounterAndClear(2);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());
//delete cube
cubeManager.dropCube(cubeName, false);
- assertTrue(cubeManager.getCube(cubeName) == null);
- assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
+
+ assertTrue(cubeManager.getCube(cubeName) == null);
+ assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
-
final String cubeDescName = "test_cube_desc";
cubeDesc.setName(cubeDescName);
cubeDesc.setLastModified(0);
@@ -245,7 +261,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));
-
cubeDesc.setNotifyList(Arrays.asList("test@email", "test@email", "test@email"));
cubeDescManager.updateCubeDesc(cubeDesc);
assertEquals(1, broadcaster.getCounterAndClear());
@@ -259,8 +274,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
-
-
getStore().deleteResource("/cube/a_whole_new_cube.json");
}
@@ -273,7 +286,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
return tableDesc;
}
-
@Test
public void testMetaCRUD() throws Exception {
final MetadataManager metadataManager = MetadataManager.getInstance(configA);
@@ -291,9 +303,6 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertNotNull(metadataManager.getTableDesc(tableDesc.getIdentity()));
assertNotNull(metadataManagerB.getTableDesc(tableDesc.getIdentity()));
-
-
-
final String dataModelName = "test_data_model";
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc("test_kylin_left_join_model_desc");
dataModelDesc.setName(dataModelName);
@@ -302,7 +311,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
assertTrue(metadataManagerB.getDataModelDesc(dataModelName) == null);
dataModelDesc.setName(dataModelName);
- metadataManager.createDataModelDesc(dataModelDesc,"default","ADMIN");
+ metadataManager.createDataModelDesc(dataModelDesc, "default", "ADMIN");
//one for data model creation, one for project meta update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9c3715fe/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 88e31f4..4cf3952 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -18,17 +18,16 @@
package org.apache.kylin.rest.service;
-import java.net.UnknownHostException;
-import java.util.List;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.net.UnknownHostException;
+import java.util.List;
/**
* @author xduo
@@ -52,7 +51,6 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);
cubeService.isCubeDescEditable(cube.getDescriptor());
- cubeService.isCubeEditable(cube);
cubes = cubeService.getCubes(null, null,null, 1, 0);
Assert.assertTrue(cubes.size() == 1);
[09/32] incubator-kylin git commit: KYLIN-693 character all build
cube/ii test cases
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f280df69/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a72b59d..81f1d97 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>16</version>
- </parent>
+ </parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
@@ -101,8 +101,10 @@
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
<sonar.jacoco.reportPath>${project.basedir}/../target/jacoco.exec</sonar.jacoco.reportPath>
<sonar.language>java</sonar.language>
- <sonar.jacoco.excludes>org/apache/kylin/**/tools/**:**/*CLI.java:net/hydromatic/optiq/**:org/eigenbase/sql2rel/**</sonar.jacoco.excludes>
-
+ <sonar.jacoco.excludes>
+ org/apache/kylin/**/tools/**:**/*CLI.java:net/hydromatic/optiq/**:org/eigenbase/sql2rel/**
+ </sonar.jacoco.excludes>
+
</properties>
<licenses>
@@ -603,6 +605,8 @@
<reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
<excludes>
<exclude>**/IT*.java</exclude>
+ <exclude>**/BuildCube*.java</exclude>
+ <exclude>**/BuildII*.java</exclude>
</excludes>
<systemProperties>
<property>
@@ -620,73 +624,97 @@
</systemProperties>
</configuration>
</plugin>
-
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>integration-tests</id>
+ <phase>integration-test</phase>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/ITKafka*.java</exclude>
+ </excludes>
+ <systemProperties>
+ <property>
+ <name>useSandbox</name>
+ <value>true</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
</plugins>
</build>
</profile>
<profile>
- <!-- This profile adds/overrides few features of the 'apache-release'
- profile in the parent pom. -->
- <id>apache-release</id>
- <build>
- <plugins>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <!-- Exclude files/folders for apache release -->
- <excludes>
- <exclude>*.md</exclude>
- <exclude>docs/website</exclude>
- <exclude>docs/*.md</exclude>
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <version>1.6</version>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>net.ju-n.maven.plugins</groupId>
- <artifactId>checksum-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>artifacts</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <algorithms>
- <algorithm>MD5</algorithm>
- <algorithm>SHA-1</algorithm>
- </algorithms>
- <failOnError>false</failOnError>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
+ <!-- This profile adds/overrides few features of the 'apache-release'
+ profile in the parent pom. -->
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <!-- Apache-RAT checks for files without headers.
+ If run on a messy developer's sandbox, it will fail.
+ This serves as a reminder to only build a release in a clean
+ sandbox! -->
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <!-- Exclude files/folders for apache release -->
+ <excludes>
+ <exclude>*.md</exclude>
+ <exclude>docs/website</exclude>
+ <exclude>docs/*.md</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-core</artifactId>
+ <version>1.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>net.ju-n.maven.plugins</groupId>
+ <artifactId>checksum-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>artifacts</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <algorithms>
+ <algorithm>MD5</algorithm>
+ <algorithm>SHA-1</algorithm>
+ </algorithms>
+ <failOnError>false</failOnError>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
[22/32] incubator-kylin git commit: KYLIN-793 modify all xmls
Posted by ma...@apache.org.
KYLIN-793 modify all xmls
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d864469
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d864469
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d864469
Branch: refs/heads/0.8.0
Commit: 7d8644691ca94bc5b525eb8043fa037fd76e4d6d
Parents: 2964a03
Author: honma <ho...@ebay.com>
Authored: Tue May 26 11:00:26 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../sandbox/capacity-scheduler.xml | 121 ++-
examples/test_case_data/sandbox/core-site.xml | 182 ++--
.../test_case_data/sandbox/hadoop-policy.xml | 74 +-
.../test_case_data/sandbox/hbase-policy.xml | 26 +-
examples/test_case_data/sandbox/hbase-site.xml | 232 +++--
examples/test_case_data/sandbox/hdfs-site.xml | 318 +++----
examples/test_case_data/sandbox/hive-site.xml | 953 +++++++++----------
examples/test_case_data/sandbox/mapred-site.xml | 286 +++---
examples/test_case_data/sandbox/yarn-site.xml | 609 ++++++------
9 files changed, 1323 insertions(+), 1478 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index 7b4a367..9c97e43 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -1,99 +1,94 @@
-<!--Tue Dec 16 19:08:08 2014-->
- <configuration>
-
+<!--Mon May 25 01:30:55 2015-->
+<configuration>
+
<property>
- <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
- <value>100</value>
+ <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.5</value>
+ <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+ <value>0.2</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.maximum-applications</name>
- <value>10000</value>
+ <name>yarn.scheduler.capacity.maximum-applications</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.node-locality-delay</name>
- <value>40</value>
+ <name>yarn.scheduler.capacity.node-locality-delay</name>
+ <value>40</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.resource-calculator</name>
- <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
- <value>*</value>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
- <value>-1</value>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+ <value>-1</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
- <value>-1</value>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+ <value>-1</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
- <value>*</value>
+ <name>yarn.scheduler.capacity.root.acl_administer_queue</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.capacity</name>
- <value>100</value>
+ <name>yarn.scheduler.capacity.root.capacity</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
- <value> </value>
+ <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+ <value> </value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
- <value>*</value>
+ <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
- <value>*</value>
+ <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.capacity</name>
- <value>100</value>
+ <name>yarn.scheduler.capacity.root.default.capacity</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.maximum-am-resource-percent</name>
- <value>0.5</value>
+ <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
- <value>100</value>
+ <name>yarn.scheduler.capacity.root.default.state</name>
+ <value>RUNNING</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.state</name>
- <value>RUNNING</value>
+ <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+ <value>1</value>
</property>
-
+
<property>
- <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
- <value>1</value>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ <value>default</value>
</property>
-
- <property>
- <name>yarn.scheduler.capacity.root.queues</name>
- <value>default</value>
- </property>
-
- </configuration>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 1b3a3ee..e2b3fbe 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -1,152 +1,106 @@
-<!--Mon Jan 19 07:28:25 2015-->
- <configuration>
-
+<!--Mon May 25 01:30:55 2015-->
+<configuration>
+
<property>
- <name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
+ <name>fs.defaultFS</name>
+ <value>hdfs://sandbox.hortonworks.com:8020</value>
+ <final>true</final>
</property>
-
+
<property>
- <name>fs.trash.interval</name>
- <value>360</value>
+ <name>fs.trash.interval</name>
+ <value>360</value>
</property>
-
+
<property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>hadoop.http.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.hcat.groups</name>
+ <value>users</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.falcon.hosts</name>
- <value>*</value>
+ <name>hadoop.proxyuser.hcat.hosts</name>
+ <value>sandbox.hortonworks.com</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.hive.groups</name>
+ <value>users</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hbase.hosts</name>
- <value>*</value>
+ <name>hadoop.proxyuser.hive.hosts</name>
+ <value>sandbox.hortonworks.com</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>hadoop.security.auth_to_local</name>
+ <value>
+ RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
+ RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
+ RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
+ RULE:[2:$1@$0](hm@.*)s/.*/hbase/
+ RULE:[2:$1@$0](rs@.*)s/.*/hbase/
+ DEFAULT</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hive.hosts</name>
- <value>*</value>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hue.groups</name>
- <value>*</value>
+ <name>io.file.buffer.size</name>
+ <value>131072</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.hue.hosts</name>
- <value>*</value>
+ <name>io.serializations</name>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>ipc.client.connect.max.retries</name>
+ <value>50</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>ipc.client.connection.maxidletime</name>
+ <value>30000</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.root.groups</name>
- <value>*</value>
+ <name>ipc.client.idlethreshold</name>
+ <value>8000</value>
</property>
-
+
<property>
- <name>hadoop.proxyuser.root.hosts</name>
- <value>*</value>
+ <name>ipc.server.tcpnodelay</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hadoop.security.auth_to_local</name>
- <value>
- RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
- RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
- RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
- RULE:[2:$1@$0](hm@.*)s/.*/hbase/
- RULE:[2:$1@$0](rs@.*)s/.*/hbase/
- DEFAULT
- </value>
+ <name>mapreduce.jobtracker.webinterface.trusted</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
+ <name>proxyuser_group</name>
+ <value>users</value>
</property>
-
- <property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
- </property>
-
- <property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
- </property>
-
- <property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
- </property>
-
- <property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
- </property>
-
- <property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
- </property>
-
- <property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
- </property>
-
- <property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
- </property>
-
- <property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
- </property>
-
- </configuration>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml
index 0f7e0b9..3f08465 100644
--- a/examples/test_case_data/sandbox/hadoop-policy.xml
+++ b/examples/test_case_data/sandbox/hadoop-policy.xml
@@ -1,59 +1,59 @@
-<!--Tue Dec 16 19:07:40 2014-->
- <configuration>
-
+<!--Mon May 25 01:30:15 2015-->
+<configuration>
+
<property>
- <name>security.admin.operations.protocol.acl</name>
- <value>hadoop</value>
+ <name>security.admin.operations.protocol.acl</name>
+ <value>hadoop</value>
</property>
-
+
<property>
- <name>security.client.datanode.protocol.acl</name>
- <value>*</value>
+ <name>security.client.datanode.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.datanode.protocol.acl</name>
- <value>*</value>
+ <name>security.datanode.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.inter.datanode.protocol.acl</name>
- <value>*</value>
+ <name>security.inter.datanode.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.inter.tracker.protocol.acl</name>
- <value>*</value>
+ <name>security.inter.tracker.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.job.client.protocol.acl</name>
- <value>*</value>
+ <name>security.job.client.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.job.task.protocol.acl</name>
- <value>*</value>
+ <name>security.job.task.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.namenode.protocol.acl</name>
- <value>*</value>
+ <name>security.namenode.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.refresh.policy.protocol.acl</name>
- <value>hadoop</value>
+ <name>security.refresh.policy.protocol.acl</name>
+ <value>hadoop</value>
</property>
-
+
<property>
- <name>security.refresh.usertogroups.mappings.protocol.acl</name>
- <value>hadoop</value>
+ <name>security.refresh.usertogroups.mappings.protocol.acl</name>
+ <value>hadoop</value>
</property>
-
- </configuration>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml
index 36756b8..6549239 100644
--- a/examples/test_case_data/sandbox/hbase-policy.xml
+++ b/examples/test_case_data/sandbox/hbase-policy.xml
@@ -1,19 +1,19 @@
-<!--Mon Jan 19 07:29:07 2015-->
- <configuration>
-
+<!--Mon May 25 01:30:31 2015-->
+<configuration>
+
<property>
- <name>security.admin.protocol.acl</name>
- <value>*</value>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
</property>
-
+
<property>
- <name>security.masterregion.protocol.acl</name>
- <value>*</value>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
</property>
-
- </configuration>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 9fd97c9..b63cc90 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -1,194 +1,174 @@
-<!--Mon Jan 19 07:29:07 2015-->
- <configuration>
-
+<!--Mon May 25 01:30:31 2015-->
+<configuration>
+
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
-
+
<property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
</property>
-
+
<property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.coprocessor.master.classes</name>
+ <value></value>
</property>
-
+
<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.coprocessor.region.classes</name>
+ <value></value>
</property>
-
+
<property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
+ <name>hbase.hregion.majorcompaction</name>
+ <value>604800000</value>
</property>
-
+
<property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
+ <name>hbase.hregion.majorcompaction.jitter</name>
+ <value>0.50</value>
</property>
-
+
<property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
+ <name>hbase.hregion.max.filesize</name>
+ <value>10737418240</value>
</property>
-
+
<property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value>4</value>
</property>
-
+
<property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value>134217728</value>
</property>
-
+
<property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
- <value>true</value>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hbase.hstore.blockingStoreFiles</name>
- <value>10</value>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <value>10</value>
</property>
-
+
<property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value>3</value>
</property>
-
+
<property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local</value>
</property>
-
+
<property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
+ <name>hbase.master.info.bindAddress</name>
+ <value>0.0.0.0</value>
</property>
-
+
<property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
+ <name>hbase.master.info.port</name>
+ <value>60010</value>
</property>
-
+
<property>
- <name>hbase.master.port</name>
- <value>60000</value>
+ <name>hbase.master.port</name>
+ <value>60000</value>
</property>
-
+
<property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
</property>
-
+
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ <value>0.4</value>
</property>
-
+
<property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <name>hbase.regionserver.handler.count</name>
+ <value>60</value>
</property>
-
+
<property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <name>hbase.regionserver.info.port</name>
+ <value>60030</value>
</property>
-
+
<property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
</property>
-
+
<property>
- <name>hbase.rpc.engine</name>
- <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
+ <name>hbase.security.authentication</name>
+ <value>simple</value>
</property>
-
+
<property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
+ <name>hbase.security.authorization</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hbase.security.authentication</name>
- <value>simple</value>
+ <name>hbase.superuser</name>
+ <value>hbase</value>
</property>
-
+
<property>
- <name>hbase.security.authorization</name>
- <value>true</value>
+ <name>hbase.tmp.dir</name>
+ <value>/hadoop/hbase</value>
</property>
-
+
<property>
- <name>hbase.superuser</name>
- <value>hbase</value>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
-
+
<property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <name>hbase.zookeeper.quorum</name>
+ <value>sandbox.hortonworks.com</value>
</property>
-
+
<property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
+ <name>hbase.zookeeper.useMulti</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hfile.block.cache.size</name>
+ <value>0.40</value>
</property>
-
+
<property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
+ <name>zookeeper.session.timeout</name>
+ <value>30000</value>
</property>
-
+
<property>
- <name>hbase_master_heapsize</name>
- <value>250</value>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase-unsecure</value>
</property>
-
- <property>
- <name>hbase_regionserver_heapsize</name>
- <value>250</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
- </property>
-
- </configuration>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index f8347dc..c9a3a6a 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -1,265 +1,241 @@
-<!--Mon Jan 19 07:29:07 2015-->
- <configuration>
-
+<!--Mon May 25 01:30:31 2015-->
+<configuration>
+
<property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
+ <name>dfs.block.access.token.enable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.block.size</name>
- <value>34217472</value>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>120</value>
</property>
-
+
<property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
</property>
-
+
<property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
+ <name>dfs.client.read.shortcircuit.streams.cache.size</name>
+ <value>4096</value>
</property>
-
+
<property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
+ <name>dfs.cluster.administrators</name>
+ <value> hdfs</value>
</property>
-
+
<property>
- <name>dfs.cluster.administrators</name>
- <value> hdfs</value>
+ <name>dfs.datanode.address</name>
+ <value>0.0.0.0:50010</value>
</property>
-
+
<property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
+ <name>dfs.datanode.balance.bandwidthPerSec</name>
+ <value>6250000</value>
</property>
-
+
<property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
+ <name>dfs.datanode.data.dir</name>
+ <value>/hadoop/hdfs/data</value>
</property>
-
+
<property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
+ <name>dfs.datanode.data.dir.perm</name>
+ <value>750</value>
</property>
-
+
<property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
+ <name>dfs.datanode.du.reserved</name>
+ <value>1073741824</value>
</property>
-
+
<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
+ <name>dfs.datanode.failed.volumes.tolerated</name>
+ <value>0</value>
</property>
-
+
<property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
+ <name>dfs.datanode.http.address</name>
+ <value>0.0.0.0:50075</value>
</property>
-
+
<property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
+ <name>dfs.datanode.https.address</name>
+ <value>0.0.0.0:50475</value>
</property>
-
+
<property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
+ <name>dfs.datanode.ipc.address</name>
+ <value>0.0.0.0:8010</value>
</property>
-
+
<property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
+ <name>dfs.datanode.max.transfer.threads</name>
+ <value>16384</value>
</property>
-
+
<property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
-
+
<property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
+ <name>dfs.heartbeat.interval</name>
+ <value>3</value>
</property>
-
+
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <name>dfs.hosts.exclude</name>
+ <value>/etc/hadoop/conf/dfs.exclude</value>
</property>
-
+
<property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <name>dfs.http.policy</name>
+ <value>HTTP_ONLY</value>
</property>
-
+
<property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
+ <name>dfs.https.port</name>
+ <value>50470</value>
</property>
-
+
<property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
+ <name>dfs.journalnode.edits.dir</name>
+ <value>/hadoop/hdfs/journalnode</value>
</property>
-
+
<property>
- <name>dfs.https.port</name>
- <value>50470</value>
+ <name>dfs.journalnode.http-address</name>
+ <value>0.0.0.0:8480</value>
</property>
-
+
<property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
+ <name>dfs.namenode.accesstime.precision</name>
+ <value>0</value>
</property>
-
+
<property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
+ <name>dfs.namenode.avoid.read.stale.datanode</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
+ <name>dfs.namenode.avoid.write.stale.datanode</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
+ <name>dfs.namenode.checkpoint.dir</name>
+ <value>/hadoop/hdfs/namesecondary</value>
</property>
-
+
<property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
+ <name>dfs.namenode.checkpoint.edits.dir</name>
+ <value>${dfs.namenode.checkpoint.dir}</value>
</property>
-
+
<property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
+ <name>dfs.namenode.checkpoint.period</name>
+ <value>21600</value>
</property>
-
+
<property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
+ <name>dfs.namenode.checkpoint.txns</name>
+ <value>1000000</value>
</property>
-
+
<property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
+ <name>dfs.namenode.handler.count</name>
+ <value>40</value>
</property>
-
+
<property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
+ <name>dfs.namenode.http-address</name>
+ <value>sandbox.hortonworks.com:50070</value>
+ <final>true</final>
</property>
-
+
<property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
+ <name>dfs.namenode.https-address</name>
+ <value>sandbox.hortonworks.com:50470</value>
</property>
-
+
<property>
- <name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
+ <name>dfs.namenode.name.dir</name>
+ <value>/hadoop/hdfs/namenode</value>
</property>
-
+
<property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
+ <name>dfs.namenode.name.dir.restore</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
+ <name>dfs.namenode.safemode.threshold-pct</name>
+ <value>1.0f</value>
</property>
-
+
<property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
+ <name>dfs.namenode.secondary.http-address</name>
+ <value>sandbox.hortonworks.com:50090</value>
</property>
-
+
<property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <name>dfs.namenode.stale.datanode.interval</name>
+ <value>30000</value>
</property>
-
+
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
+ <name>dfs.namenode.startup.delay.block.deletion.sec</name>
+ <value>3600</value>
</property>
-
+
<property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
+ <name>dfs.namenode.write.stale.datanode.ratio</name>
+ <value>1.0f</value>
</property>
-
+
<property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
+ <name>dfs.permissions.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
+ <name>dfs.permissions.superusergroup</name>
+ <value>hdfs</value>
</property>
-
+
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
+ <name>dfs.replication</name>
+ <value>3</value>
</property>
-
+
<property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
+ <name>dfs.replication.max</name>
+ <value>50</value>
</property>
-
+
<property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
+ <name>dfs.support.append</name>
+ <value>true</value>
+ <final>true</final>
</property>
-
+
<property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
+ <name>dfs.webhdfs.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>dfs.replication</name>
- <value>1</value>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
</property>
-
- <property>
- <name>dfs.replication.max</name>
- <value>50</value>
- </property>
-
- <property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.webhdfs.enabled</name>
- <value>true</value>
- <final>true</final>
- </property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
- </configuration>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index 40afbf6..7efaf4f 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -1,794 +1,779 @@
-<!--Tue Dec 16 19:33:41 2014-->
- <configuration>
-
+<!--Mon May 25 01:23:17 2015-->
+<configuration>
+
<property>
- <name>ambari.hive.db.schema.name</name>
- <value>hive</value>
+ <name>ambari.hive.db.schema.name</name>
+ <value>hive</value>
</property>
-
+
<property>
- <name>datanucleus.cache.level2.type</name>
- <value>none</value>
+ <name>datanucleus.cache.level2.type</name>
+ <value>none</value>
</property>
-
+
<property>
- <name>hive.auto.convert.join</name>
- <value>true</value>
+ <name>hive.auto.convert.join</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.auto.convert.join.noconditionaltask</name>
- <value>true</value>
+ <name>hive.auto.convert.join.noconditionaltask</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>1000000000</value>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>1251999744</value>
</property>
-
+
<property>
- <name>hive.auto.convert.sortmerge.join</name>
- <value>true</value>
+ <name>hive.auto.convert.sortmerge.join</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.auto.convert.sortmerge.join.noconditionaltask</name>
- <value>true</value>
+ <name>hive.auto.convert.sortmerge.join.noconditionaltask</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
- <value>false</value>
+ <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.cbo.enable</name>
- <value>true</value>
+ <name>hive.cbo.enable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.cli.print.header</name>
- <value>false</value>
+ <name>hive.cli.print.header</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.cluster.delegation.token.store.class</name>
- <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value>
+ <name>hive.cluster.delegation.token.store.class</name>
+ <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value>
</property>
-
+
<property>
- <name>hive.cluster.delegation.token.store.zookeeper.connectString</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <name>hive.cluster.delegation.token.store.zookeeper.connectString</name>
+ <value>sandbox.hortonworks.com:2181</value>
</property>
-
+
<property>
- <name>hive.cluster.delegation.token.store.zookeeper.znode</name>
- <value>/hive/cluster/delegation</value>
+ <name>hive.cluster.delegation.token.store.zookeeper.znode</name>
+ <value>/hive/cluster/delegation</value>
</property>
-
+
<property>
- <name>hive.compactor.abortedtxn.threshold</name>
- <value>1000</value>
+ <name>hive.compactor.abortedtxn.threshold</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>hive.compactor.check.interval</name>
- <value>300s</value>
+ <name>hive.compactor.check.interval</name>
+ <value>300L</value>
</property>
-
+
<property>
- <name>hive.compactor.delta.num.threshold</name>
- <value>10</value>
+ <name>hive.compactor.delta.num.threshold</name>
+ <value>10</value>
</property>
-
+
<property>
- <name>hive.compactor.delta.pct.threshold</name>
- <value>0.1f</value>
+ <name>hive.compactor.delta.pct.threshold</name>
+ <value>0.1f</value>
</property>
-
+
<property>
- <name>hive.compactor.initiator.on</name>
- <value>true</value>
+ <name>hive.compactor.initiator.on</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.compactor.worker.threads</name>
- <value>0</value>
+ <name>hive.compactor.worker.threads</name>
+ <value>0</value>
</property>
-
+
<property>
- <name>hive.compactor.worker.timeout</name>
- <value>86400s</value>
+ <name>hive.compactor.worker.timeout</name>
+ <value>86400L</value>
</property>
-
+
<property>
- <name>hive.compute.query.using.stats</name>
- <value>true</value>
+ <name>hive.compute.query.using.stats</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.conf.restricted.list</name>
- <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value>
+ <name>hive.conf.restricted.list</name>
+ <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value>
</property>
-
+
<property>
- <name>hive.convert.join.bucket.mapjoin.tez</name>
- <value>false</value>
+ <name>hive.convert.join.bucket.mapjoin.tez</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.enforce.bucketing</name>
- <value>true</value>
+ <name>hive.enforce.bucketing</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.enforce.sorting</name>
- <value>true</value>
+ <name>hive.enforce.sorting</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.enforce.sortmergebucketmapjoin</name>
- <value>true</value>
+ <name>hive.enforce.sortmergebucketmapjoin</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.exec.compress.intermediate</name>
- <value>false</value>
+ <name>hive.exec.compress.intermediate</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.exec.compress.output</name>
- <value>false</value>
+ <name>hive.exec.compress.output</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.exec.dynamic.partition</name>
- <value>true</value>
+ <name>hive.exec.dynamic.partition</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.exec.dynamic.partition.mode</name>
- <value>nonstrict</value>
+ <name>hive.exec.dynamic.partition.mode</name>
+ <value>nonstrict</value>
</property>
-
+
<property>
- <name>hive.exec.failure.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ <name>hive.exec.failure.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
</property>
-
+
<property>
- <name>hive.exec.max.created.files</name>
- <value>100000</value>
+ <name>hive.exec.max.created.files</name>
+ <value>100000</value>
</property>
-
+
<property>
- <name>hive.exec.max.dynamic.partitions</name>
- <value>5000</value>
+ <name>hive.exec.max.dynamic.partitions</name>
+ <value>5000</value>
</property>
-
+
<property>
- <name>hive.exec.max.dynamic.partitions.pernode</name>
- <value>2000</value>
+ <name>hive.exec.max.dynamic.partitions.pernode</name>
+ <value>2000</value>
</property>
-
+
<property>
- <name>hive.exec.orc.compression.strategy</name>
- <value>SPEED</value>
+ <name>hive.exec.orc.compression.strategy</name>
+ <value>SPEED</value>
</property>
-
+
<property>
- <name>hive.exec.orc.default.compress</name>
- <value>ZLIB</value>
+ <name>hive.exec.orc.default.compress</name>
+ <value>ZLIB</value>
</property>
-
+
<property>
- <name>hive.exec.orc.default.stripe.size</name>
- <value>67108864</value>
+ <name>hive.exec.orc.default.stripe.size</name>
+ <value>67108864</value>
</property>
-
+
<property>
- <name>hive.exec.parallel</name>
- <value>false</value>
+ <name>hive.exec.parallel</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.exec.parallel.thread.number</name>
- <value>8</value>
+ <name>hive.exec.parallel.thread.number</name>
+ <value>8</value>
</property>
-
+
<property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
</property>
-
+
<property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
</property>
-
+
<property>
- <name>hive.exec.reducers.bytes.per.reducer</name>
- <value>67108864</value>
+ <name>hive.exec.reducers.bytes.per.reducer</name>
+ <value>67108864</value>
</property>
-
+
<property>
- <name>hive.exec.reducers.max</name>
- <value>1009</value>
+ <name>hive.exec.reducers.max</name>
+ <value>1009</value>
</property>
-
+
<property>
- <name>hive.exec.scratchdir</name>
- <value>/tmp/hive</value>
+ <name>hive.exec.scratchdir</name>
+ <value>/tmp/hive</value>
</property>
-
+
<property>
- <name>hive.exec.submit.local.task.via.child</name>
- <value>true</value>
+ <name>hive.exec.submit.local.task.via.child</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.exec.submitviachild</name>
- <value>false</value>
+ <name>hive.exec.submitviachild</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.execution.engine</name>
- <value>mr</value>
+ <name>hive.execution.engine</name>
+ <value>mr</value>
</property>
-
+
<property>
- <name>hive.fetch.task.aggr</name>
- <value>false</value>
+ <name>hive.fetch.task.aggr</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.fetch.task.conversion</name>
- <value>more</value>
+ <name>hive.fetch.task.conversion</name>
+ <value>more</value>
</property>
-
+
<property>
- <name>hive.fetch.task.conversion.threshold</name>
- <value>1073741824</value>
+ <name>hive.fetch.task.conversion.threshold</name>
+ <value>1073741824</value>
</property>
-
+
<property>
- <name>hive.heapsize</name>
- <value>250</value>
+ <name>hive.heapsize</name>
+ <value>1024</value>
</property>
-
+
<property>
- <name>hive.limit.optimize.enable</name>
- <value>true</value>
+ <name>hive.limit.optimize.enable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.limit.pushdown.memory.usage</name>
- <value>0.04</value>
+ <name>hive.limit.pushdown.memory.usage</name>
+ <value>0.04</value>
</property>
-
+
<property>
- <name>hive.map.aggr</name>
- <value>true</value>
+ <name>hive.map.aggr</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.map.aggr.hash.force.flush.memory.threshold</name>
- <value>0.9</value>
+ <name>hive.map.aggr.hash.force.flush.memory.threshold</name>
+ <value>0.9</value>
</property>
-
+
<property>
- <name>hive.map.aggr.hash.min.reduction</name>
- <value>0.5</value>
+ <name>hive.map.aggr.hash.min.reduction</name>
+ <value>0.5</value>
</property>
-
+
<property>
- <name>hive.map.aggr.hash.percentmemory</name>
- <value>0.5</value>
+ <name>hive.map.aggr.hash.percentmemory</name>
+ <value>0.5</value>
</property>
-
+
<property>
- <name>hive.mapjoin.bucket.cache.size</name>
- <value>10000</value>
+ <name>hive.mapjoin.bucket.cache.size</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>hive.mapjoin.optimized.hashtable</name>
- <value>true</value>
+ <name>hive.mapjoin.optimized.hashtable</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.mapred.reduce.tasks.speculative.execution</name>
- <value>false</value>
+ <name>hive.mapred.reduce.tasks.speculative.execution</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.merge.mapfiles</name>
- <value>true</value>
+ <name>hive.merge.mapfiles</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.merge.mapredfiles</name>
- <value>false</value>
+ <name>hive.merge.mapredfiles</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.merge.orcfile.stripe.level</name>
- <value>true</value>
+ <name>hive.merge.orcfile.stripe.level</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.merge.rcfile.block.level</name>
- <value>true</value>
+ <name>hive.merge.rcfile.block.level</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.merge.size.per.task</name>
- <value>256000000</value>
+ <name>hive.merge.size.per.task</name>
+ <value>256000000</value>
</property>
-
+
<property>
- <name>hive.merge.smallfiles.avgsize</name>
- <value>16000000</value>
+ <name>hive.merge.smallfiles.avgsize</name>
+ <value>16000000</value>
</property>
-
+
<property>
- <name>hive.merge.tezfiles</name>
- <value>false</value>
+ <name>hive.merge.tezfiles</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.metastore.authorization.storage.checks</name>
- <value>false</value>
+ <name>hive.metastore.authorization.storage.checks</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.metastore.cache.pinobjtypes</name>
- <value>Table,Database,Type,FieldSchema,Order</value>
+ <name>hive.metastore.cache.pinobjtypes</name>
+ <value>Table,Database,Type,FieldSchema,Order</value>
</property>
-
+
<property>
- <name>hive.metastore.client.connect.retry.delay</name>
- <value>5s</value>
+ <name>hive.metastore.client.connect.retry.delay</name>
+ <value>5s</value>
</property>
-
+
<property>
- <name>hive.metastore.client.socket.timeout</name>
- <value>1800s</value>
+ <name>hive.metastore.client.socket.timeout</name>
+ <value>1800s</value>
</property>
-
+
<property>
- <name>hive.metastore.connect.retries</name>
- <value>24</value>
+ <name>hive.metastore.connect.retries</name>
+ <value>24</value>
</property>
-
+
<property>
- <name>hive.metastore.execute.setugi</name>
- <value>true</value>
+ <name>hive.metastore.execute.setugi</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.metastore.failure.retries</name>
- <value>24</value>
+ <name>hive.metastore.failure.retries</name>
+ <value>24</value>
</property>
-
+
<property>
- <name>hive.metastore.kerberos.keytab.file</name>
- <value>/etc/security/keytabs/hive.service.keytab</value>
+ <name>hive.metastore.kerberos.keytab.file</name>
+ <value>/etc/security/keytabs/hive.service.keytab</value>
</property>
-
+
<property>
- <name>hive.metastore.kerberos.principal</name>
- <value>hive/_HOST@EXAMPLE.COM</value>
+ <name>hive.metastore.kerberos.principal</name>
+ <value>hive/_HOST@EXAMPLE.COM</value>
</property>
-
+
<property>
- <name>hive.metastore.pre.event.listeners</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value>
+ <name>hive.metastore.pre.event.listeners</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value>
</property>
-
+
<property>
- <name>hive.metastore.sasl.enabled</name>
- <value>false</value>
+ <name>hive.metastore.sasl.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.metastore.server.max.threads</name>
- <value>100000</value>
+ <name>hive.metastore.server.max.threads</name>
+ <value>100000</value>
</property>
-
+
<property>
- <name>hive.metastore.uris</name>
- <value>thrift://sandbox.hortonworks.com:9083</value>
+ <name>hive.metastore.uris</name>
+ <value>thrift://sandbox.hortonworks.com:9083</value>
</property>
-
+
<property>
- <name>hive.metastore.warehouse.dir</name>
- <value>/apps/hive/warehouse</value>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>/apps/hive/warehouse</value>
</property>
-
+
<property>
- <name>hive.optimize.bucketmapjoin</name>
- <value>true</value>
+ <name>hive.optimize.bucketmapjoin</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.bucketmapjoin.sortedmerge</name>
- <value>false</value>
+ <name>hive.optimize.bucketmapjoin.sortedmerge</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.optimize.constant.propagation</name>
- <value>true</value>
+ <name>hive.optimize.constant.propagation</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.index.filter</name>
- <value>true</value>
+ <name>hive.optimize.index.filter</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.mapjoin.mapreduce</name>
- <value>true</value>
+ <name>hive.optimize.mapjoin.mapreduce</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.metadataonly</name>
- <value>true</value>
+ <name>hive.optimize.metadataonly</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.null.scan</name>
- <value>true</value>
+ <name>hive.optimize.null.scan</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.reducededuplication</name>
- <value>true</value>
+ <name>hive.optimize.reducededuplication</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.optimize.reducededuplication.min.reducer</name>
- <value>4</value>
+ <name>hive.optimize.reducededuplication.min.reducer</name>
+ <value>4</value>
</property>
-
+
<property>
- <name>hive.optimize.sort.dynamic.partition</name>
- <value>false</value>
+ <name>hive.optimize.sort.dynamic.partition</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.orc.compute.splits.num.threads</name>
- <value>10</value>
+ <name>hive.orc.compute.splits.num.threads</name>
+ <value>10</value>
</property>
-
+
<property>
- <name>hive.orc.splits.include.file.footer</name>
- <value>false</value>
+ <name>hive.orc.splits.include.file.footer</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.prewarm.enabled</name>
- <value>false</value>
+ <name>hive.prewarm.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.prewarm.numcontainers</name>
- <value>10</value>
+ <name>hive.prewarm.numcontainers</name>
+ <value>10</value>
</property>
-
+
<property>
- <name>hive.security.authenticator.manager</name>
- <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value>
+ <name>hive.security.authenticator.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value>
</property>
-
+
<property>
- <name>hive.security.authorization.enabled</name>
- <value>false</value>
+ <name>hive.security.authorization.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.security.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value>
+ <name>hive.security.authorization.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value>
</property>
-
+
<property>
- <name>hive.security.metastore.authenticator.manager</name>
- <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value>
+ <name>hive.security.metastore.authenticator.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value>
</property>
-
+
<property>
- <name>hive.security.metastore.authorization.auth.reads</name>
- <value>true</value>
+ <name>hive.security.metastore.authorization.auth.reads</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
+ <name>hive.security.metastore.authorization.manager</name>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
</property>
-
+
<property>
- <name>hive.server2.allow.user.substitution</name>
- <value>true</value>
+ <name>hive.server2.allow.user.substitution</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.server2.authentication</name>
- <value>NONE</value>
+ <name>hive.server2.authentication</name>
+ <value>NONE</value>
</property>
-
+
<property>
- <name>hive.server2.authentication.spnego.keytab</name>
- <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <name>hive.server2.authentication.spnego.keytab</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
</property>
-
+
<property>
- <name>hive.server2.authentication.spnego.principal</name>
- <value>/etc/security/keytabs/spnego.service.keytab</value>
+ <name>hive.server2.authentication.spnego.principal</name>
+ <value>/etc/security/keytabs/spnego.service.keytab</value>
</property>
-
+
<property>
- <name>hive.server2.enable.doAs</name>
- <value>false</value>
+ <name>hive.server2.enable.doAs</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.server2.enable.impersonation</name>
- <value>true</value>
+ <name>hive.server2.enable.impersonation</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.server2.logging.operation.enabled</name>
- <value>true</value>
+ <name>hive.server2.logging.operation.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.server2.logging.operation.log.location</name>
- <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
+ <name>hive.server2.logging.operation.log.location</name>
+ <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
</property>
-
+
<property>
- <name>hive.server2.support.dynamic.service.discovery</name>
- <value>true</value>
+ <name>hive.server2.support.dynamic.service.discovery</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.server2.table.type.mapping</name>
- <value>CLASSIC</value>
+ <name>hive.server2.table.type.mapping</name>
+ <value>CLASSIC</value>
</property>
-
+
<property>
- <name>hive.server2.tez.default.queues</name>
- <value>default</value>
+ <name>hive.server2.tez.default.queues</name>
+ <value>default</value>
</property>
-
+
<property>
- <name>hive.server2.tez.initialize.default.sessions</name>
- <value>false</value>
+ <name>hive.server2.tez.initialize.default.sessions</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.server2.tez.sessions.per.default.queue</name>
- <value>1</value>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ <value>1</value>
</property>
-
+
<property>
- <name>hive.server2.thrift.http.path</name>
- <value>cliservice</value>
+ <name>hive.server2.thrift.http.path</name>
+ <value>cliservice</value>
</property>
-
+
<property>
- <name>hive.server2.thrift.http.port</name>
- <value>10001</value>
+ <name>hive.server2.thrift.http.port</name>
+ <value>10001</value>
</property>
-
+
<property>
- <name>hive.server2.thrift.max.worker.threads</name>
- <value>500</value>
+ <name>hive.server2.thrift.max.worker.threads</name>
+ <value>500</value>
</property>
-
+
<property>
- <name>hive.server2.thrift.port</name>
- <value>10000</value>
+ <name>hive.server2.thrift.port</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>hive.server2.thrift.sasl.qop</name>
- <value>auth</value>
+ <name>hive.server2.thrift.sasl.qop</name>
+ <value>auth</value>
</property>
-
+
<property>
- <name>hive.server2.transport.mode</name>
- <value>binary</value>
+ <name>hive.server2.transport.mode</name>
+ <value>binary</value>
</property>
-
+
<property>
- <name>hive.server2.use.SSL</name>
- <value>false</value>
+ <name>hive.server2.use.SSL</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.server2.zookeeper.namespace</name>
- <value>hiveserver2</value>
+ <name>hive.server2.zookeeper.namespace</name>
+ <value>hiveserver2</value>
</property>
-
+
<property>
- <name>hive.smbjoin.cache.rows</name>
- <value>10000</value>
+ <name>hive.smbjoin.cache.rows</name>
+ <value>10000</value>
</property>
-
+
<property>
- <name>hive.stats.autogather</name>
- <value>true</value>
+ <name>hive.stats.autogather</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.stats.dbclass</name>
- <value>fs</value>
+ <name>hive.stats.dbclass</name>
+ <value>fs</value>
</property>
-
+
<property>
- <name>hive.stats.fetch.column.stats</name>
- <value>false</value>
+ <name>hive.stats.fetch.column.stats</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.stats.fetch.partition.stats</name>
- <value>true</value>
+ <name>hive.stats.fetch.partition.stats</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.support.concurrency</name>
- <value>true</value>
+ <name>hive.support.concurrency</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.tez.auto.reducer.parallelism</name>
- <value>false</value>
+ <name>hive.tez.auto.reducer.parallelism</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.tez.container.size</name>
- <value>250</value>
+ <name>hive.tez.container.size</name>
+ <value>3584</value>
</property>
-
+
<property>
- <name>hive.tez.cpu.vcores</name>
- <value>-1</value>
+ <name>hive.tez.cpu.vcores</name>
+ <value>-1</value>
</property>
-
+
<property>
- <name>hive.tez.dynamic.partition.pruning</name>
- <value>true</value>
+ <name>hive.tez.dynamic.partition.pruning</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.tez.dynamic.partition.pruning.max.data.size</name>
- <value>104857600</value>
+ <name>hive.tez.dynamic.partition.pruning.max.data.size</name>
+ <value>104857600</value>
</property>
-
+
<property>
- <name>hive.tez.dynamic.partition.pruning.max.event.size</name>
- <value>1048576</value>
+ <name>hive.tez.dynamic.partition.pruning.max.event.size</name>
+ <value>1048576</value>
</property>
-
+
<property>
- <name>hive.tez.input.format</name>
- <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value>
+ <name>hive.tez.input.format</name>
+ <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value>
</property>
-
+
<property>
- <name>hive.tez.java.opts</name>
- <value>-server -Xmx200m -Djava.net.preferIPv4Stack=true</value>
+ <name>hive.tez.java.opts</name>
+ <value>-server -Xmx2868m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps</value>
</property>
-
+
<property>
- <name>hive.tez.log.level</name>
- <value>INFO</value>
+ <name>hive.tez.log.level</name>
+ <value>INFO</value>
</property>
-
+
<property>
- <name>hive.tez.max.partition.factor</name>
- <value>2.0</value>
+ <name>hive.tez.max.partition.factor</name>
+ <value>2.0</value>
</property>
-
+
<property>
- <name>hive.tez.min.partition.factor</name>
- <value>0.25</value>
+ <name>hive.tez.min.partition.factor</name>
+ <value>0.25</value>
</property>
-
+
<property>
- <name>hive.tez.smb.number.waves</name>
- <value>0.5</value>
+ <name>hive.tez.smb.number.waves</name>
+ <value>0.5</value>
</property>
-
+
<property>
- <name>hive.txn.manager</name>
- <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
+ <name>hive.txn.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager</value>
</property>
-
+
<property>
- <name>hive.txn.max.open.batch</name>
- <value>1000</value>
+ <name>hive.txn.max.open.batch</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>hive.txn.timeout</name>
- <value>300</value>
+ <name>hive.txn.timeout</name>
+ <value>300</value>
</property>
-
+
<property>
- <name>hive.user.install.directory</name>
- <value>/user/</value>
+ <name>hive.user.install.directory</name>
+ <value>/user/</value>
</property>
-
+
<property>
- <name>hive.users.in.admin.role</name>
- <value>hue,hive</value>
+ <name>hive.vectorized.execution.enabled</name>
+ <value>true</value>
</property>
-
+
<property>
- <name>hive.vectorized.execution.enabled</name>
- <value>true</value>
+ <name>hive.vectorized.execution.reduce.enabled</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>hive.vectorized.execution.reduce.enabled</name>
- <value>false</value>
+ <name>hive.vectorized.groupby.checkinterval</name>
+ <value>4096</value>
</property>
-
+
<property>
- <name>hive.vectorized.groupby.checkinterval</name>
- <value>4096</value>
+ <name>hive.vectorized.groupby.flush.percent</name>
+ <value>0.1</value>
</property>
-
+
<property>
- <name>hive.vectorized.groupby.flush.percent</name>
- <value>0.1</value>
+ <name>hive.vectorized.groupby.maxentries</name>
+ <value>100000</value>
</property>
-
+
<property>
- <name>hive.vectorized.groupby.maxentries</name>
- <value>100000</value>
+ <name>hive.zookeeper.client.port</name>
+ <value>2181</value>
</property>
-
+
<property>
- <name>hive.zookeeper.client.port</name>
- <value>2181</value>
+ <name>hive.zookeeper.namespace</name>
+ <value>hive_zookeeper_namespace</value>
</property>
-
+
<property>
- <name>hive.zookeeper.namespace</name>
- <value>hive_zookeeper_namespace</value>
+ <name>hive.zookeeper.quorum</name>
+ <value>sandbox.hortonworks.com:2181</value>
</property>
-
+
<property>
- <name>hive.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.jdbc.Driver</value>
</property>
-
+
<property>
- <name>hive_metastore_user_passwd</name>
- <value>hive</value>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true</value>
</property>
-
+
<property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>com.mysql.jdbc.Driver</value>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>hive</value>
</property>
-
- <property>
- <name>javax.jdo.option.ConnectionPassword</name>
- <value>hive</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>hive</value>
- </property>
-
- </configuration>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d864469/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index ee3837c..f6a1c2e 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -1,239 +1,219 @@
-<!--Tue Dec 16 19:08:08 2014-->
- <configuration>
-
+<!--Mon May 25 01:30:55 2015-->
+<configuration>
+
<property>
- <name>io.sort.mb</name>
- <value>64</value>
+ <name>mapreduce.admin.map.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
</property>
-
+
<property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
+ <name>mapreduce.admin.reduce.child.java.opts</name>
+ <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
</property>
-
+
<property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
</property>
-
+
<property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
</property>
-
+
<property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ <name>mapreduce.application.classpath</name>
+ <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
-
+
<property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
+ <name>mapreduce.application.framework.path</name>
+ <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
</property>
-
+
<property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ <name>mapreduce.cluster.administrators</name>
+ <value> hadoop</value>
</property>
-
+
<property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
</property>
-
+
<property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*,$PWD/mr-framework/hadoop/share/hadoop/common/*,$PWD/mr-framework/hadoop/share/hadoop/common/lib/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar,/etc/hadoop/conf/secure</value>
+ <name>mapreduce.job.emit-timeline-data</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.05</value>
</property>
-
+
<property>
- <name>mapreduce.cluster.administrators</name>
- <value> hadoop</value>
+ <name>mapreduce.jobhistory.address</name>
+ <value>sandbox.hortonworks.com:10020</value>
</property>
-
+
<property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
+ <name>mapreduce.jobhistory.bind-host</name>
+ <value>0.0.0.0</value>
</property>
-
+
<property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
+ <name>mapreduce.jobhistory.done-dir</name>
+ <value>/mr-history/done</value>
</property>
-
+
<property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
+ <name>mapreduce.jobhistory.intermediate-done-dir</name>
+ <value>/mr-history/tmp</value>
</property>
-
+
<property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>sandbox.hortonworks.com:19888</value>
</property>
-
+
<property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Xmx2867m</value>
</property>
-
+
<property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
+ <name>mapreduce.map.log.level</name>
+ <value>INFO</value>
</property>
-
+
<property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
+ <name>mapreduce.map.memory.mb</name>
+ <value>3584</value>
</property>
-
+
<property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
+ <name>mapreduce.map.output.compress</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx200m</value>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.7</value>
</property>
-
+
<property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>mapreduce.map.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
</property>
-
+
<property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
+ <name>mapreduce.reduce.input.buffer.percent</name>
+ <value>0.0</value>
</property>
-
+
<property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Xmx2867m</value>
</property>
-
+
<property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
+ <name>mapreduce.reduce.log.level</name>
+ <value>INFO</value>
</property>
-
+
<property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>3584</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
+ <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
+ <value>1</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
+ <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
+ <value>1000</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
+ <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
+ <value>30000</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
+ <value>0.7</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>30</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
+ <name>mapreduce.shuffle.port</name>
+ <value>13562</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>100</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>1024</value>
</property>
-
+
<property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
+ <name>mapreduce.task.timeout</name>
+ <value>300000</value>
</property>
-
+
<property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value>-Dhdp.version=${hdp.version}</value>
</property>
-
+
<property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Xmx2867m -Dhdp.version=${hdp.version}</value>
</property>
-
+
<property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>64</value>
+ <name>yarn.app.mapreduce.am.log.level</name>
+ <value>INFO</value>
</property>
-
+
<property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>3584</value>
</property>
-
+
<property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
</property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
-
- </configuration>
+
+</configuration>
[12/32] incubator-kylin git commit: KYLIN-697 fix mock htable bug
Posted by ma...@apache.org.
KYLIN-697 fix mock htable bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/216427c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/216427c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/216427c9
Branch: refs/heads/0.8.0
Commit: 216427c9874bf26e60b244e495738a99be60b12c
Parents: d22836d
Author: honma <ho...@ebay.com>
Authored: Mon May 25 10:32:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:20:40 2015 +0800
----------------------------------------------------------------------
common/src/main/java/org/apache/kylin/common/util/MockHTable.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/216427c9/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
index daa068d..c60a7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
+++ b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
@@ -680,7 +680,6 @@ public class MockHTable implements HTableInterface {
public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey, R responsePrototype, Batch.Callback<R> callback) throws ServiceException, Throwable {
throw new NotImplementedException();
-
}
@Override
[02/32] incubator-kylin git commit: KYLIN-697 fix mock htable bug
Posted by ma...@apache.org.
KYLIN-697 fix mock htable bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/43ce8bc2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/43ce8bc2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/43ce8bc2
Branch: refs/heads/0.8.0
Commit: 43ce8bc2528867121c8cb02a381e326079f69e3d
Parents: 8e1d6f5
Author: honma <ho...@ebay.com>
Authored: Mon May 25 10:32:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:11:52 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/MockHTable.java | 7 +-
.../kylin/job/tools/HBaseRowDigestTest.java | 84 --------------------
2 files changed, 5 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43ce8bc2/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
index d54fe4b..c60a7a5 100644
--- a/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
+++ b/common/src/main/java/org/apache/kylin/common/util/MockHTable.java
@@ -516,7 +516,11 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
+ if (kv.isDeleteFamily()) {
+ data.get(row).get(kv.getFamily()).clear();
+ } else {
+ data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
+ }
}
if (data.get(row).get(family).isEmpty()) {
data.get(row).remove(family);
@@ -676,7 +680,6 @@ public class MockHTable implements HTableInterface {
public <R extends Message> void batchCoprocessorService(Descriptors.MethodDescriptor methodDescriptor, Message request, byte[] startKey, byte[] endKey, R responsePrototype, Batch.Callback<R> callback) throws ServiceException, Throwable {
throw new NotImplementedException();
-
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43ce8bc2/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java b/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
deleted file mode 100644
index 95e4b7f..0000000
--- a/job/src/test/java/org/apache/kylin/job/tools/HBaseRowDigestTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.apache.kylin.job.tools;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- */
-@Ignore
-public class HBaseRowDigestTest extends HBaseMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- private static final byte[] CF = "f".getBytes();
- private static final byte[] QN = "c".getBytes();
- static ImmutableBytesWritable k = new ImmutableBytesWritable();
- static ImmutableBytesWritable v = new ImmutableBytesWritable();
-
- @Test
- public static void test() throws IOException {
- String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = null;
- HTableInterface table = null;
- try {
- conn = HBaseConnection.get(hbaseUrl);
- table = conn.getTable("KYLIN_II_YTYWP3CQGJ");
- ResultScanner scanner = table.getScanner(CF, QN);
- StringBuffer sb = new StringBuffer();
- while (true) {
- Result r = scanner.next();
- if (r == null)
- break;
-
- Cell[] cells = r.rawCells();
- Cell c = cells[0];
-
- k.set(c.getRowArray(), c.getRowOffset(), c.getRowLength());
- v.set(c.getValueArray(), c.getValueOffset(), c.getValueLength());
-
- byte[] row = k.copyBytes();
- byte[] value = v.copyBytes();
- // byte[] row = r.getRow();
- // byte[] value = r.getValue(CF, QN);
- //
- sb.append("row length: " + row.length + "\r\n");
- sb.append(BytesUtil.toReadableText(row) + "\r\n");
- sb.append("value length: " + value.length + "\r\n");
- sb.append(BytesUtil.toReadableText(value) + "\r\n");
- }
- System.out.println(sb.toString());
- FileUtils.writeStringToFile(new File("/Users/honma/Desktop/a3"), sb.toString());
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (table != null)
- table.close();
- if (conn != null)
- conn.close();
- }
-
- }
-}
[11/32] incubator-kylin git commit: KYLIN-697 non-integration tests
passed after merging codes
Posted by ma...@apache.org.
KYLIN-697 non-integration tests passed after merging codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4e35d9f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4e35d9f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4e35d9f3
Branch: refs/heads/0.8.0
Commit: 4e35d9f3c95c0a984467f4aa6a051b7f523c73e8
Parents: 4b6e6e2
Author: honma <ho...@ebay.com>
Authored: Mon May 25 13:19:51 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:17:16 2015 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/AbstractRestCache.java | 1 +
.../kylin/common/restclient/SingleValueCache.java | 13 ++++++++-----
.../java/org/apache/kylin/cube/CubeManagerTest.java | 6 +++---
.../apache/kylin/rest/service/CacheServiceTest.java | 1 +
4 files changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e35d9f3/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java b/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
index d9e98fd..c728ff2 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/AbstractRestCache.java
@@ -24,6 +24,7 @@ package org.apache.kylin.common.restclient;
*/
public abstract class AbstractRestCache<K, V> {
+
protected final Broadcaster.TYPE syncType;
protected AbstractRestCache(Broadcaster.TYPE syncType) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e35d9f3/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index fb44206..00f8a3e 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -43,10 +43,12 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- //enforce all cache changes coming from REST
- //final V result = innerCache.put(key, value);
+ boolean exists = innerCache.containsKey(key);
+ //The put operation will be duplicated when REST request is received.
+ //It is intended so because many test cases does not have REST env
+ innerCache.put(key, value);
- if (!innerCache.containsKey(key)) {
+ if (!exists) {
syncRemote(key, Broadcaster.EVENT.CREATE);
} else {
syncRemote(key, Broadcaster.EVENT.UPDATE);
@@ -59,8 +61,9 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void remove(K key) {
if (innerCache.containsKey(key)) {
- //enforce all cache changes coming from REST
- //innerCache.remove(key);
+ //The remove operation will be duplicated when REST request is received.
+ //It is intended so because many test cases does not have REST env
+ innerCache.remove(key);
syncRemote(key, Broadcaster.EVENT.DROP);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e35d9f3/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 8970295..3b0bd3f 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.*;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.JsonUtil;
@@ -31,6 +29,8 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* @author yangli9
*/
@@ -75,7 +75,7 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));
- CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", true);
+ CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
assertTrue(createdCube == droppedCube);
assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4e35d9f3/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 9f0598d..795afaf 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -125,6 +125,7 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
counter.set(0L);
+ createTestMetadata();
}
@After
[16/32] incubator-kylin git commit: KYLIN-697 category
IIStreamBuilderTest to IIKafka*
Posted by ma...@apache.org.
KYLIN-697 category IIStreamBuilderTest to IIKafka*
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ee8959c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ee8959c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ee8959c2
Branch: refs/heads/0.8.0
Commit: ee8959c24562657bacf6bb3e8402d323f6a27dba
Parents: 7fcfada
Author: honma <ho...@ebay.com>
Authored: Tue May 26 11:25:22 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue May 26 23:21:27 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/ITIIStreamBuilderTest.java | 108 -------------------
.../job/ITKafkaBasedIIStreamBuilderTest.java | 108 +++++++++++++++++++
2 files changed, 108 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ee8959c2/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
deleted file mode 100644
index 2edf457..0000000
--- a/job/src/test/java/org/apache/kylin/job/ITIIStreamBuilderTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *
- * contributor license agreements. See the NOTICE file distributed with
- *
- * this work for additional information regarding copyright ownership.
- *
- * The ASF licenses this file to You under the Apache License, Version 2.0
- *
- * (the "License"); you may not use this file except in compliance with
- *
- * the License. You may obtain a copy of the License at
- *
- *
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- *
- *
- * Unless required by applicable law or agreed to in writing, software
- *
- * distributed under the License is distributed on an "AS IS" BASIS,
- *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *
- * See the License for the specific language governing permissions and
- *
- * limitations under the License.
- *
- * /
- */
-
-package org.apache.kylin.job;
-
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
-import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- */
-public class ITIIStreamBuilderTest {
-
- private static final Logger logger = LoggerFactory.getLogger(ITIIStreamBuilderTest.class);
-
- private KylinConfig kylinConfig;
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
- }
-
- @AfterClass
- public static void afterClass() throws Exception {
-// backup();
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- exportHBaseData();
- }
- }
-
- private static int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
-
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- }
-
- @Before
- public void before() throws Exception {
- HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- DeployUtil.initCliWorkDir();
- DeployUtil.deployMetadata();
- DeployUtil.overrideJobJarLocations();
- }
-
- @Test
- public void test() throws Exception {
- final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
- bootstrap.start("eagle", 0);
- Thread.sleep(30 * 60 * 1000);
- logger.info("time is up, stop streaming");
- bootstrap.stop();
- Thread.sleep(5 * 1000);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ee8959c2/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
new file mode 100644
index 0000000..756a38c
--- /dev/null
+++ b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -0,0 +1,108 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
+import org.apache.kylin.job.streaming.StreamingBootstrap;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ */
+public class ITKafkaBasedIIStreamBuilderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITKafkaBasedIIStreamBuilderTest.class);
+
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+// backup();
+ }
+
+ private static void backup() throws Exception {
+ int exitCode = cleanupOldStorage();
+ if (exitCode == 0) {
+ exportHBaseData();
+ }
+ }
+
+ private static int cleanupOldStorage() throws Exception {
+ String[] args = {"--delete", "true"};
+
+ int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
+ return exitCode;
+ }
+
+ private static void exportHBaseData() throws IOException {
+ ExportHBaseData export = new ExportHBaseData();
+ export.exportTables();
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+ }
+
+ @Test
+ public void test() throws Exception {
+ final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
+ bootstrap.start("eagle", 0);
+ Thread.sleep(30 * 60 * 1000);
+ logger.info("time is up, stop streaming");
+ bootstrap.stop();
+ Thread.sleep(5 * 1000);
+ }
+}