You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/07 13:20:37 UTC
[02/28] incubator-kylin git commit: KYLIN-956 set gzip as default
compression for test case and snappy for binary tar
KYLIN-956 set gzip as default compression for test case and snappy for binary tar
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c73f5dea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c73f5dea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c73f5dea
Branch: refs/heads/2.x-staging
Commit: c73f5dea4171a98a5daf3275e4feed921909a083
Parents: 42fa50d
Author: honma <ho...@ebay.com>
Authored: Mon Aug 31 11:06:08 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 1 14:14:03 2015 +0800
----------------------------------------------------------------------
bin/setenv.sh | 4 ++--
.../engine/spark/BuildCubeWithSparkTest.java | 17 ++++++++-------
.../test_case_data/localmeta/kylin.properties | 2 +-
.../localmeta/kylin_hive_conf.xml | 2 ++
.../test_case_data/localmeta/kylin_job_conf.xml | 4 ++--
.../test_case_data/sandbox/kylin.properties | 2 +-
.../test_case_data/sandbox/kylin_hive_conf.xml | 4 +++-
.../test_case_data/sandbox/kylin_job_conf.xml | 3 ++-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
.../kylin/job/BuildCubeWithStreamTest.java | 1 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 1 -
.../apache/kylin/job/BuildIIWithStreamTest.java | 2 +-
.../kylin/storage/hbase/steps/BulkLoadJob.java | 1 +
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 22 +++++++-------------
14 files changed, 33 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/setenv.sh b/bin/setenv.sh
index 5acb287..f4cce2d 100755
--- a/bin/setenv.sh
+++ b/bin/setenv.sh
@@ -18,7 +18,7 @@ then
echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings"
+ echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
fi
if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
@@ -34,5 +34,5 @@ then
echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to enable LZO compression, etc. "
+ echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to specify your own native path"
fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index 178b10f..d24cc79 100644
--- a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -18,7 +18,13 @@
package org.apache.kylin.engine.spark;
-import com.google.common.collect.Lists;
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
@@ -42,12 +48,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-
-import static org.junit.Assert.assertEquals;
+import com.google.common.collect.Lists;
public class BuildCubeWithSparkTest {
@@ -120,7 +121,7 @@ public class BuildCubeWithSparkTest {
waitForJob(cubingJob.getId());
assertEquals(ExecutableState.SUCCEED, jobService.getOutput(cubingJob.getId()).getState());
}
-
+
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
// remove all existing segments
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 6240862..bdd6113 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -37,7 +37,7 @@ kylin.job.concurrent.max.limit=10
kylin.job.yarn.app.rest.check.interval.seconds=10
#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=snappy
+kylin.hbase.default.compression.codec=gzip
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_hive_conf.xml b/examples/test_case_data/localmeta/kylin_hive_conf.xml
index 821b6ee..3cb5242 100644
--- a/examples/test_case_data/localmeta/kylin_hive_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_hive_conf.xml
@@ -24,6 +24,7 @@
<description>enable map-side join</description>
</property>
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
@@ -34,6 +35,7 @@
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
+ -->
<property>
<name>hive.merge.mapfiles</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_job_conf.xml b/examples/test_case_data/localmeta/kylin_job_conf.xml
index 0f5bf30..10dfdb7 100644
--- a/examples/test_case_data/localmeta/kylin_job_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_job_conf.xml
@@ -11,6 +11,7 @@
</property>
+ <!--
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
@@ -42,8 +43,7 @@
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
-
-
+ -->
<property>
<name>mapreduce.job.max.split.locations</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index ef0c288..ce599ea 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -49,7 +49,7 @@ kylin.job.hbase.region.cut.large=50
kylin.job.yarn.app.rest.check.interval.seconds=10
#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=snappy
+kylin.hbase.default.compression.codec=gzip
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index eab0399..faa00d9 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -24,6 +24,7 @@
<description>enable map-side join</description>
</property>
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
@@ -34,7 +35,8 @@
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
-
+ -->
+
<property>
<name>hive.merge.mapfiles</name>
<value>true</value>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index 2d1fe10..69def6e 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -11,6 +11,7 @@
</property>
+ <!--
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
@@ -42,7 +43,7 @@
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
-
+-->
<property>
<name>mapreduce.job.max.split.locations</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index f2b49c5..611bf8b 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -43,7 +43,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index d3f84a0..87b684e 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -75,7 +75,6 @@ public class BuildCubeWithStreamTest {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
-
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 1cf1a0f..ff88667 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -237,7 +237,6 @@ public class BuildIIWithEngineTest {
return exitCode;
}
-
public static void main(String[] args) throws Exception {
BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 360ad8b..e3a12ea 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -120,7 +120,7 @@ public class BuildIIWithStreamTest {
JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
final String uuid = UUID.randomUUID().toString();
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig,uuid));
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid));
String insertDataHqls;
try {
insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 5059a37..ec4a8a9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -48,6 +48,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
Options options = new Options();
try {
+
options.addOption(OPTION_INPUT_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_CUBE_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 7d45fd6..f717d82 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -51,10 +51,12 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
+@Ignore
public class TestFuzzyRowFilterV2EndToEnd {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static byte fuzzyValue = (byte) 63;
@@ -75,8 +77,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt("hbase.client.scanner.caching", 1000);
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
- ConstantSizeRegionSplitPolicy.class.getName());
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
// set no splits
conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
@@ -111,8 +112,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public void testEndToEnd() throws Exception {
String cf = "f";
- HTable ht =
- TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+ HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
// 10 byte row key - (2 bytes 4 bytes 4 bytes)
// 4 byte qualifier
@@ -250,8 +250,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public void testFilterList() throws Exception {
String cf = "f";
String table = "TestFuzzyRowFiltersInFilterList";
- HTable ht =
- TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+ HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
// 10 byte row key - (2 bytes 4 bytes 4 bytes)
// 4 byte qualifier
@@ -276,8 +275,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
p.setDurability(Durability.SKIP_WAL);
p.add(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
- LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
- + Bytes.toStringBinary(cq));
+ LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
}
}
}
@@ -320,8 +318,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
}
- private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
- throws IOException {
+ private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
String cf = "f";
Scan scan = new Scan();
scan.addFamily(cf.getBytes());
@@ -334,8 +331,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
long timeBeforeScan = System.currentTimeMillis();
while ((result = scanner.next()) != null) {
for (Cell kv : result.listCells()) {
- LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
- + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
+ LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
results.add(kv);
}
}
@@ -348,5 +344,3 @@ public class TestFuzzyRowFilterV2EndToEnd {
assertEquals(expectedSize, results.size());
}
}
-
-