You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/09/29 03:04:05 UTC
[14/27] incubator-kylin git commit: Remove hardcoded “hdp.version” from codes;
Remove hardcoded “hdp.version” from codes;
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/297cb233
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/297cb233
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/297cb233
Branch: refs/heads/2.x-staging
Commit: 297cb2336eca1f92b6314b742ebca8f2e5bbb9f3
Parents: b3f93b4
Author: shaofengshi <sh...@apache.org>
Authored: Mon Sep 21 10:14:37 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Sep 29 09:01:53 2015 +0800
----------------------------------------------------------------------
.../engine/spark/BuildCubeWithSparkTest.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../kylin/job/BuildCubeWithStreamTest.java | 2 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 2 +-
.../apache/kylin/job/BuildIIWithStreamTest.java | 2 +-
.../kylin/job/DeployLocalMetaToRemoteTest.java | 2 +-
.../job/ITKafkaBasedIIStreamBuilderTest.java | 85 ++++++++++++++++++++
pom.xml | 4 -
.../hbase/steps/SandboxMetastoreCLI.java | 2 +-
9 files changed, 92 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index d24cc79..de60504 100644
--- a/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/assembly/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -78,7 +78,7 @@ public class BuildCubeWithSparkTest {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index b2b46c1..d6d9738 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -85,7 +85,7 @@ public class BuildCubeWithEngineTest {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index c0e7978..0b42f12 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -75,7 +75,7 @@ public class BuildCubeWithStreamTest {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index fecb106..a824cf5 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -93,7 +93,7 @@ public class BuildIIWithEngineTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 256297e..795441a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -98,7 +98,7 @@ public class BuildIIWithStreamTest {
public static void beforeClass() throws Exception {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index 7f12069..9d3000e 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -45,7 +45,7 @@ public class DeployLocalMetaToRemoteTest {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/assembly/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/assembly/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
new file mode 100644
index 0000000..739f35a
--- /dev/null
+++ b/assembly/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -0,0 +1,85 @@
+/*
+ *
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *
+ * contributor license agreements. See the NOTICE file distributed with
+ *
+ * this work for additional information regarding copyright ownership.
+ *
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ *
+ * (the "License"); you may not use this file except in compliance with
+ *
+ * the License. You may obtain a copy of the License at
+ *
+ *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ *
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ *
+ * limitations under the License.
+ *
+ * /
+ */
+
+package org.apache.kylin.job;
+
+import java.io.File;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractKylinTestCase;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.job.streaming.StreamingBootstrap;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+@Ignore("this test case will break existing metadata store")
+public class ITKafkaBasedIIStreamBuilderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(ITKafkaBasedIIStreamBuilderTest.class);
+
+ private KylinConfig kylinConfig;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ }
+
+ @Before
+ public void before() throws Exception {
+ HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ DeployUtil.initCliWorkDir();
+ DeployUtil.deployMetadata();
+ DeployUtil.overrideJobJarLocations();
+ }
+
+ @Test
+ public void test() throws Exception {
+ final StreamingBootstrap bootstrap = StreamingBootstrap.getInstance(kylinConfig);
+ bootstrap.start("eagle", 0);
+ Thread.sleep(30 * 60 * 1000);
+ logger.info("time is up, stop streaming");
+ bootstrap.stop();
+ Thread.sleep(5 * 1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fa33d2e..471c319 100644
--- a/pom.xml
+++ b/pom.xml
@@ -700,10 +700,6 @@
<name>useSandbox</name>
<value>true</value>
</property>
- <property>
- <name>hdp.version</name>
- <value>2.2.4.2-2</value>
- </property>
</systemProperties>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/297cb233/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index 21a6e43..ac9d0c2 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -41,7 +41,7 @@ public class SandboxMetastoreCLI {
logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+ //System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
if (args.length < 2) {
printUsage();