You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/24 07:07:26 UTC
incubator-kylin git commit: KYLIN-1096 Deprecate minicluster
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging dff3c03f6 -> bfe6d15e7
KYLIN-1096 Deprecate minicluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bfe6d15e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bfe6d15e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bfe6d15e
Branch: refs/heads/2.x-staging
Commit: bfe6d15e72082b2b917643ed040da5199a4a533a
Parents: dff3c03
Author: shaofengshi <sh...@apache.org>
Authored: Tue Nov 24 14:05:34 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Nov 24 14:07:02 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithStreamTest.java | 23 ---
.../org/apache/kylin/job/ExportHBaseData.java | 160 ------------------
.../hive/ITHiveSourceTableLoaderTest.java | 3 -
.../common/util/AbstractKylinTestCase.java | 4 -
.../kylin/metadata/MetadataUpgradeTest.java | 114 -------------
.../minicluster/hbase-export.tar.gz | Bin 22048217 -> 0 bytes
.../test_case_data/minicluster/kylin.properties | 63 -------
.../minicluster/kylin_job_conf.lzo_disabled.xml | 76 ---------
.../minicluster/kylin_job_conf.lzo_enabled.xml | 76 ---------
.../minicluster/kylin_job_conf.xml | 76 ---------
pom.xml | 45 -----
.../storage/hbase/common/MiniClusterTest.java | 63 -------
.../hbase/steps/HBaseMetadataTestCase.java | 28 +---
.../hbase/steps/HBaseMiniclusterHelper.java | 167 -------------------
.../storage/hbase/steps/HbaseImporter.java | 121 --------------
15 files changed, 5 insertions(+), 1014 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 63bba70..9e60622 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -100,32 +100,9 @@ public class BuildCubeWithStreamTest {
@AfterClass
public static void afterClass() throws Exception {
- backup();
HBaseMetadataTestCase.staticCleanupTestMetadata();
}
- private static int cleanupOldStorage() throws Exception {
- String[] args = { "--delete", "true" };
- int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
- return exitCode;
- }
-
- private static void backup() throws Exception {
- int exitCode = cleanupOldStorage();
- if (exitCode == 0) {
- String exportHTables = System.getProperty("kylinExportHTables");
- if (Boolean.parseBoolean(exportHTables) == true) {
- exportHBaseData();
- }
- }
- }
-
- private static void exportHBaseData() throws IOException {
- ExportHBaseData export = new ExportHBaseData();
- export.exportTables();
- export.tearDown();
- }
-
@Test
public void test() throws Exception {
logger.info("start time:" + startTime + " end time:" + endTime + " batch interval:" + batchInterval + " batch count:" + ((endTime - startTime) / batchInterval));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/assembly/src/test/java/org/apache/kylin/job/ExportHBaseData.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/assembly/src/test/java/org/apache/kylin/job/ExportHBaseData.java
deleted file mode 100644
index 5eb7485..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/ExportHBaseData.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.SSHClient;
-import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.steps.HBaseMiniclusterHelper;
-
-public class ExportHBaseData {
-
- KylinConfig kylinConfig;
- HTableDescriptor[] allTables;
- Configuration config;
- HBaseAdmin hbase;
- CliCommandExecutor cli;
- String exportHdfsFolder;
- String exportLocalFolderParent;
- String exportLocalFolder;
- String backupArchive;
- String tableNameBase;
- long currentTIME;
-
- public ExportHBaseData() {
- try {
- setup();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private void setup() throws IOException {
-
- KylinConfig.destoryInstance();
- System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cli = kylinConfig.getCliCommandExecutor();
-
- currentTIME = System.currentTimeMillis();
- exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/";
- exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/";
- exportLocalFolder = exportLocalFolderParent + currentTIME + "/";
- backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz";
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = metadataUrl.substring(0, cut);
- String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- HConnection conn = HBaseConnection.get(hbaseUrl);
- try {
- hbase = new HBaseAdmin(conn);
- config = hbase.getConfiguration();
- allTables = hbase.listTables();
- } catch (IOException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- public void tearDown() {
-
- // cleanup hdfs
- try {
- if (cli != null && exportHdfsFolder != null) {
- cli.execute("hadoop fs -rm -r " + exportHdfsFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- // cleanup sandbox disk
- try {
- if (cli != null && exportLocalFolder != null) {
- cli.execute("rm -r " + exportLocalFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- // delete archive file on sandbox
- try {
- if (cli != null && backupArchive != null) {
- cli.execute("rm " + backupArchive);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void exportTables() throws IOException {
- cli.execute("mkdir -p " + exportLocalFolderParent);
-
- for (HTableDescriptor table : allTables) {
- String tName = table.getNameAsString();
- if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
- continue;
-
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName);
- }
-
- cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent);
- cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME);
- downloadToLocal();
- }
-
- public void downloadToLocal() throws IOException {
- String localArchive = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
-
- if (kylinConfig.getRunAsRemoteCommand()) {
- SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliPort(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword());
- try {
- ssh.scpFileToLocal(backupArchive, localArchive);
- } catch (Exception e) {
- e.printStackTrace();
- }
- } else {
- FileUtils.copyFile(new File(backupArchive), new File(localArchive));
- }
- }
-
- public static void main(String[] args) {
- ExportHBaseData export = new ExportHBaseData();
- try {
- export.exportTables();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- export.tearDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
index 8218d51..23780cf 100644
--- a/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ b/assembly/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
@@ -43,9 +43,6 @@ public class ITHiveSourceTableLoaderTest extends HBaseMetadataTestCase {
@Test
public void test() throws IOException {
- if (!useSandbox())
- return;
-
KylinConfig config = getTestConfig();
String[] toLoad = new String[] { "DEFAULT.TEST_KYLIN_FACT", "EDW.TEST_CAL_DT" };
Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(toLoad, config);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
index 42e059b..d517930 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java
@@ -30,10 +30,6 @@ public abstract class AbstractKylinTestCase {
public static final String LOCALMETA_TEST_DATA = "../examples/test_case_data/localmeta";
- public static final String LOCALMETA_TEST_DATA_V1 = "../examples/test_case_data/localmeta_v1";
-
- public static final String MINICLUSTER_TEST_DATA = "../examples/test_case_data/minicluster";
-
public static final String SANDBOX_TEST_DATA = "../examples/test_case_data/sandbox";
public static final String[] SERVICES_WITH_CACHE = {//
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
deleted file mode 100644
index e37aabc..0000000
--- a/core-cube/src/test/java/org/apache/kylin/metadata/MetadataUpgradeTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata;
-
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test the data model upgrade
- * @author shaoshi
- *
- */
-@Ignore("Not needed, the migrate and test has been moved to CubeMetadataUpgrade.java")
-public class MetadataUpgradeTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- staticCreateTestMetadata(LOCALMETA_TEST_DATA_V1);
- }
-
- @After
- public void after() throws Exception {
- //this.cleanupTestMetadata();
- }
-
- @Test
- public void testCubeDescUpgrade() throws Exception {
-
- String[] sampleCubeDescs = new String[] { "test_kylin_cube_with_slr_desc", "test_kylin_cube_with_slr_left_join_desc", "test_kylin_cube_without_slr_desc", "test_kylin_cube_without_slr_left_join_desc" };
-
- for (String name : sampleCubeDescs)
- checkCubeDesc(name);
-
- }
-
- @Test
- public void testTableDescUpgrade() throws Exception {
-
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- TableDesc fact = metaMgr.getTableDesc("default.test_kylin_fact");
-
- @SuppressWarnings("deprecation")
- String oldResLocation = fact.getResourcePathV1();
- String newResLocation = fact.getResourcePath();
-
- ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
-
- Assert.assertTrue(store.exists(newResLocation));
- Assert.assertTrue(!store.exists(oldResLocation));
-
- String oldExdResLocation = TableDesc.concatExdResourcePath("test_kylin_fact".toUpperCase());
- String newExdResLocation = TableDesc.concatExdResourcePath("default.test_kylin_fact".toUpperCase());
-
- Assert.assertTrue(store.exists(newExdResLocation));
- Assert.assertTrue(!store.exists(oldExdResLocation));
-
- }
-
- private void checkCubeDesc(String descName) {
- CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
- CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
- Assert.assertNotNull(cubedesc1);
- DataModelDesc model = cubedesc1.getModel();
- Assert.assertNotNull(model);
- Assert.assertTrue(model.getLookups().length > 0);
-
- List<DimensionDesc> dims = cubedesc1.getDimensions();
-
- Assert.assertTrue(dims.size() > 0);
-
- for (DimensionDesc dim : dims) {
- Assert.assertTrue(dim.getColumn().length > 0);
- }
-
- Assert.assertTrue(cubedesc1.getMeasures().size() > 0);
-
- CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);
-
- Assert.assertTrue(cubes.size() > 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/examples/test_case_data/minicluster/hbase-export.tar.gz
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/hbase-export.tar.gz b/examples/test_case_data/minicluster/hbase-export.tar.gz
deleted file mode 100755
index 9a327a2..0000000
Binary files a/examples/test_case_data/minicluster/hbase-export.tar.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
deleted file mode 100644
index abd5714..0000000
--- a/examples/test_case_data/minicluster/kylin.properties
+++ /dev/null
@@ -1,63 +0,0 @@
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=ADMIN:KYLIN@localhost
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=false
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=
-
-# Hive JDBC connection properties; Not in use now
-kylin.hive.jdbc.driver=org.apache.hive.jdbc.HiveDriver
-kylin.hive.jdbc.url=jdbc:hive2:///
-kylin.hive.jdbc.user=
-kylin.hive.jdbc.password=
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=10
-
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
deleted file mode 100644
index c189063..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_disabled.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml b/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
deleted file mode 100644
index c189063..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.lzo_enabled.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/examples/test_case_data/minicluster/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin_job_conf.xml b/examples/test_case_data/minicluster/kylin_job_conf.xml
deleted file mode 100644
index c189063..0000000
--- a/examples/test_case_data/minicluster/kylin_job_conf.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3c696f7..31ab924 100644
--- a/pom.xml
+++ b/pom.xml
@@ -621,51 +621,6 @@
<profiles>
<profile>
- <id>minicluster</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.16</version>
- <configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
- <excludes>
- <!--Build cube/II need to be run separately-->
- <exclude>**/BuildCube*.java</exclude>
- <exclude>**/BuildII*.java</exclude>
-
- <!--minicluster does not have kafka-->
- <exclude>**/ITKafka*.java</exclude>
-
- <!--minicluster does not have hive-->
- <exclude>**/ITSnapshotManagerTest.java</exclude>
- <exclude>**/ITHiveTableReaderTest.java</exclude>
- <exclude>**/ITTableControllerTest.java</exclude>
-
- </excludes>
- <systemProperties>
- <property>
- <name>useSandbox</name>
- <value>false</value>
- </property>
- <property>
- <name>buildCubeUsingProvidedData</name>
- <value>false</value>
- </property>
- </systemProperties>
- <argLine>-Xms1G -Xmx4G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
- </configuration>
- </plugin>
-
- </plugins>
- </build>
- </profile>
-
- <profile>
<id>sandbox</id>
<activation>
<activeByDefault>true</activeByDefault>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
deleted file mode 100644
index 949e22a..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/common/MiniClusterTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.common;
-
-import java.io.File;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-
-public class MiniClusterTest {
-
- private static HBaseTestingUtility testUtil = new HBaseTestingUtility();
-
- public static void main(String[] args) throws Exception {
-
- File miniclusterFolder = new File(AbstractKylinTestCase.MINICLUSTER_TEST_DATA);
- System.out.println("----" + miniclusterFolder.getAbsolutePath());
-
- //save the dfs data to minicluster folder
- System.setProperty("test.build.data", miniclusterFolder.getAbsolutePath());
-
- MiniHBaseCluster hbCluster = testUtil.startMiniCluster(1);
- testUtil.startMiniMapReduceCluster();
- System.out.println("Minicluster started.");
-
- Configuration conf = hbCluster.getConf();
- String host = conf.get(HConstants.ZOOKEEPER_QUORUM);
- String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- String parent = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
-
- // reduce rpc retry
- conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- conf.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "5");
- conf.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- String connectionUrl = "hbase:" + host + ":" + port + ":" + parent;
-
- System.out.println("hbase connection url:" + connectionUrl);
-
- testUtil.getDFSCluster().getFileSystem();
- testUtil.shutdownMiniMapReduceCluster();
- testUtil.shutdownMiniCluster();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java
index 08ed83a..2a0adc3 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMetadataTestCase.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase.steps;
import java.io.File;
-import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
@@ -31,12 +30,10 @@ import org.apache.kylin.common.util.ClassUtil;
public class HBaseMetadataTestCase extends AbstractKylinTestCase {
static {
- if (useSandbox()) {
- try {
- ClassUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath());
- } catch (Exception e) {
- e.printStackTrace();
- }
+ try {
+ ClassUtil.addClasspath(new File("../examples/test_case_data/sandbox/").getAbsolutePath());
+ } catch (Exception e) {
+ e.printStackTrace();
}
}
@@ -51,13 +48,7 @@ public class HBaseMetadataTestCase extends AbstractKylinTestCase {
}
public static void staticCreateTestMetadata() throws Exception {
- if (useSandbox()) {
- staticCreateTestMetadata(SANDBOX_TEST_DATA);
- } else {
- staticCreateTestMetadata(MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- }
-
+ staticCreateTestMetadata(SANDBOX_TEST_DATA);
}
public static void staticCreateTestMetadata(String kylinConfigFolder) {
@@ -69,13 +60,4 @@ public class HBaseMetadataTestCase extends AbstractKylinTestCase {
}
- public static boolean useSandbox() {
- String useSandbox = System.getProperty("useSandbox");
- if (StringUtils.isEmpty(useSandbox)) {
- return true;
- }
-
- return Boolean.parseBoolean(useSandbox);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMiniclusterHelper.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMiniclusterHelper.java
deleted file mode 100644
index c62508d..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HBaseMiniclusterHelper.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.steps;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.storage.hbase.HBaseResourceStore;
-
-/**
- * a helper class to start and shutdown hbase mini cluster
- *
- * @author shaoshi
- */
-public class HBaseMiniclusterHelper {
-
- public static final String SHARED_STORAGE_PREFIX = "KYLIN_";
- public static final String CUBE_STORAGE_PREFIX = "KYLIN_";
- public static final String II_STORAGE_PREFIX = "KYLIN_II_";
- public static final String TEST_METADATA_TABLE = "kylin_metadata";
-
- private static final String hbaseTarLocation = "../examples/test_case_data/minicluster/hbase-export.tar.gz";
- private static final String iiEndpointClassName = "org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint";
-
- public static HBaseTestingUtility UTIL = new HBaseTestingUtility();
- private static volatile boolean clusterStarted = false;
- private static String hbaseconnectionUrl = "";
-
- private static final Log logger = LogFactory.getLog(HBaseMiniclusterHelper.class);
-
- static {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- shutdownMiniCluster();
- }
- });
- }
-
- /**
- * Start the minicluster; Sub-classes should invoke this in BeforeClass method.
- *
- * @throws Exception
- */
- public static void startupMinicluster() throws Exception {
-
- if (!clusterStarted) {
- synchronized (HBaseMiniclusterHelper.class) {
- if (!clusterStarted) {
- startupMiniClusterAndImportData();
- clusterStarted = true;
- }
- }
- } else {
- updateKylinConfigWithMinicluster();
- }
- }
-
- private static void updateKylinConfigWithMinicluster() {
-
- KylinConfig.getInstanceFromEnv().setMetadataUrl(TEST_METADATA_TABLE + "@" + hbaseconnectionUrl);
- KylinConfig.getInstanceFromEnv().setStorageUrl(hbaseconnectionUrl);
- }
-
- private static void startupMiniClusterAndImportData() throws Exception {
-
- logger.info("Going to start mini cluster.");
-
- if (existInClassPath(iiEndpointClassName)) {
- HBaseMiniclusterHelper.UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, iiEndpointClassName);
- }
-
- //https://issues.apache.org/jira/browse/HBASE-11711
- UTIL.getConfiguration().setInt("hbase.master.info.port", -1);//avoid port clobbering
-
- MiniHBaseCluster hbaseCluster = UTIL.startMiniCluster();
-
- Configuration config = hbaseCluster.getConf();
- String host = config.get(HConstants.ZOOKEEPER_QUORUM);
- String port = config.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- String parent = config.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
-
- // see in: https://hbase.apache.org/book.html#trouble.rs.runtime.zkexpired
- config.set("zookeeper.session.timeout", "1200000");
- config.set("hbase.zookeeper.property.tickTime", "6000");
- // reduce rpc retry
- config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
- config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- hbaseconnectionUrl = "hbase:" + host + ":" + port + ":" + parent;
- updateKylinConfigWithMinicluster();
-
- UTIL.startMiniMapReduceCluster();
-
- // create the metadata htables;
- @SuppressWarnings("unused")
- HBaseResourceStore store = new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
-
- // import the table content
- HbaseImporter.importHBaseData(hbaseTarLocation, UTIL.getConfiguration());
-
- }
-
- private static boolean existInClassPath(String className) {
- try {
- Class.forName(className);
- } catch (ClassNotFoundException e) {
- return false;
- }
- return true;
- }
-
- /**
- * Shutdown the minicluster;
- */
- public static void shutdownMiniCluster() {
-
- logger.info("Going to shutdown mini cluster.");
-
- try {
- UTIL.shutdownMiniMapReduceCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- try {
- UTIL.shutdownMiniCluster();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- HBaseMiniclusterHelper t = new HBaseMiniclusterHelper();
- logger.info(t);
- try {
- HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.MINICLUSTER_TEST_DATA);
- HBaseMiniclusterHelper.startupMinicluster();
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- HBaseMiniclusterHelper.shutdownMiniCluster();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bfe6d15e/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HbaseImporter.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HbaseImporter.java
deleted file mode 100644
index d5346d2..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/HbaseImporter.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.steps;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.mapreduce.Import;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-
-import com.google.common.base.Preconditions;
-
-/**
- */
-public class HbaseImporter {
-
- private static final Log logger = LogFactory.getLog(HbaseImporter.class);
-
- public static void importHBaseData(String hbaseTarLocation, Configuration conf) throws IOException, ClassNotFoundException, InterruptedException {
-
- if (System.getenv("JAVA_HOME") == null) {
- logger.error("Didn't find $JAVA_HOME, this will cause HBase data import failed. Please set $JAVA_HOME.");
- logger.error("Skipping table import...");
- return;
- }
-
- File exportFile = new File(hbaseTarLocation);
- if (!exportFile.exists()) {
- logger.error("Didn't find the export achieve file on " + exportFile.getAbsolutePath());
- return;
- }
-
- File folder = File.createTempFile("hbase-import", "tmp");
- if (folder.exists()) {
- FileUtils.forceDelete(folder);
- }
- folder.mkdirs();
- FileUtils.forceDeleteOnExit(folder);
-
- //TarGZUtil.uncompressTarGZ(exportFile, folder);
- FileUtil.unTar(exportFile, folder);
- String[] child = folder.list();
- Preconditions.checkState(child.length == 1);
- String backupFolderName = child[0];
- File backupFolder = new File(folder, backupFolderName);
- String[] tableNames = backupFolder.list();
-
- for (String table : tableNames) {
-
- if (!(table.equalsIgnoreCase(HBaseMiniclusterHelper.TEST_METADATA_TABLE) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) {
- continue;
- }
-
- // create the htable; otherwise the import will fail.
- if (table.startsWith(HBaseMiniclusterHelper.II_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "f");
- } else if (table.startsWith(HBaseMiniclusterHelper.CUBE_STORAGE_PREFIX)) {
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2");
- }
-
- // directly import from local fs, no need to copy to hdfs
- String importLocation = "file://" + backupFolder.getAbsolutePath() + "/" + table;
- String[] args = new String[] { table, importLocation };
- boolean result = runImport(args, conf);
- logger.info("importing table '" + table + "' with result:" + result);
-
- if (!result)
- break;
- }
-
- }
-
- private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException {
- // need to make a copy of the configuration because to make sure different temp dirs are used.
- GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args);
- Configuration newConf = opts.getConfiguration();
- args = opts.getRemainingArgs();
- Job job = Import.createSubmittableJob(newConf, args);
- job.waitForCompletion(false);
- return job.isSuccessful();
- }
-
- public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
- if (args.length != 1) {
- logger.error("Usage: HbaseImporter hbase_tar_lcoation");
- System.exit(-1);
- }
-
- logger.info("The KylinConfig being used:");
- logger.info("=================================================");
- KylinConfig.getInstanceFromEnv().printProperties();
- logger.info("=================================================");
-
- importHBaseData(args[0], HBaseConfiguration.create());
- }
-}