You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/01 08:34:14 UTC
[1/4] incubator-kylin git commit: KYLIN-956 set gzip as default
compression for test case and snappy for binary tar
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 08845a1ec -> ab2abeec5
KYLIN-956 set gzip as default compression for test case and snappy for binary tar
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c73f5dea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c73f5dea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c73f5dea
Branch: refs/heads/0.8
Commit: c73f5dea4171a98a5daf3275e4feed921909a083
Parents: 42fa50d
Author: honma <ho...@ebay.com>
Authored: Mon Aug 31 11:06:08 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 1 14:14:03 2015 +0800
----------------------------------------------------------------------
bin/setenv.sh | 4 ++--
.../engine/spark/BuildCubeWithSparkTest.java | 17 ++++++++-------
.../test_case_data/localmeta/kylin.properties | 2 +-
.../localmeta/kylin_hive_conf.xml | 2 ++
.../test_case_data/localmeta/kylin_job_conf.xml | 4 ++--
.../test_case_data/sandbox/kylin.properties | 2 +-
.../test_case_data/sandbox/kylin_hive_conf.xml | 4 +++-
.../test_case_data/sandbox/kylin_job_conf.xml | 3 ++-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
.../kylin/job/BuildCubeWithStreamTest.java | 1 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 1 -
.../apache/kylin/job/BuildIIWithStreamTest.java | 2 +-
.../kylin/storage/hbase/steps/BulkLoadJob.java | 1 +
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 22 +++++++-------------
14 files changed, 33 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/setenv.sh b/bin/setenv.sh
index 5acb287..f4cce2d 100755
--- a/bin/setenv.sh
+++ b/bin/setenv.sh
@@ -18,7 +18,7 @@ then
echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings"
+ echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
fi
if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
@@ -34,5 +34,5 @@ then
echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to enable LZO compression, etc. "
+ echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to specify your own native path"
fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index 178b10f..d24cc79 100644
--- a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -18,7 +18,13 @@
package org.apache.kylin.engine.spark;
-import com.google.common.collect.Lists;
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
@@ -42,12 +48,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.TimeZone;
-
-import static org.junit.Assert.assertEquals;
+import com.google.common.collect.Lists;
public class BuildCubeWithSparkTest {
@@ -120,7 +121,7 @@ public class BuildCubeWithSparkTest {
waitForJob(cubingJob.getId());
assertEquals(ExecutableState.SUCCEED, jobService.getOutput(cubingJob.getId()).getState());
}
-
+
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
// remove all existing segments
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 6240862..bdd6113 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -37,7 +37,7 @@ kylin.job.concurrent.max.limit=10
kylin.job.yarn.app.rest.check.interval.seconds=10
#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=snappy
+kylin.hbase.default.compression.codec=gzip
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_hive_conf.xml b/examples/test_case_data/localmeta/kylin_hive_conf.xml
index 821b6ee..3cb5242 100644
--- a/examples/test_case_data/localmeta/kylin_hive_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_hive_conf.xml
@@ -24,6 +24,7 @@
<description>enable map-side join</description>
</property>
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
@@ -34,6 +35,7 @@
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
+ -->
<property>
<name>hive.merge.mapfiles</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/localmeta/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin_job_conf.xml b/examples/test_case_data/localmeta/kylin_job_conf.xml
index 0f5bf30..10dfdb7 100644
--- a/examples/test_case_data/localmeta/kylin_job_conf.xml
+++ b/examples/test_case_data/localmeta/kylin_job_conf.xml
@@ -11,6 +11,7 @@
</property>
+ <!--
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
@@ -42,8 +43,7 @@
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
-
-
+ -->
<property>
<name>mapreduce.job.max.split.locations</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index ef0c288..ce599ea 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -49,7 +49,7 @@ kylin.job.hbase.region.cut.large=50
kylin.job.yarn.app.rest.check.interval.seconds=10
#default compression codec for htable,snappy,lzo,gzip,lz4
-kylin.hbase.default.compression.codec=snappy
+kylin.hbase.default.compression.codec=gzip
## Config for Restful APP ##
# database connection settings:
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index eab0399..faa00d9 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -24,6 +24,7 @@
<description>enable map-side join</description>
</property>
+ <!--
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
@@ -34,7 +35,8 @@
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
-
+ -->
+
<property>
<name>hive.merge.mapfiles</name>
<value>true</value>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index 2d1fe10..69def6e 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -11,6 +11,7 @@
</property>
+ <!--
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
@@ -42,7 +43,7 @@
<value>BLOCK</value>
<description>The compression type to use for job outputs</description>
</property>
-
+-->
<property>
<name>mapreduce.job.max.split.locations</name>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index f2b49c5..611bf8b 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -43,7 +43,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index d3f84a0..87b684e 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -75,7 +75,6 @@ public class BuildCubeWithStreamTest {
ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
-
}
@Before
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 1cf1a0f..ff88667 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -237,7 +237,6 @@ public class BuildIIWithEngineTest {
return exitCode;
}
-
public static void main(String[] args) throws Exception {
BuildIIWithEngineTest instance = new BuildIIWithEngineTest();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 360ad8b..e3a12ea 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -120,7 +120,7 @@ public class BuildIIWithStreamTest {
JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
final String uuid = UUID.randomUUID().toString();
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig,uuid));
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig, uuid));
String insertDataHqls;
try {
insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
index 5059a37..ec4a8a9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java
@@ -48,6 +48,7 @@ public class BulkLoadJob extends AbstractHadoopJob {
Options options = new Options();
try {
+
options.addOption(OPTION_INPUT_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_CUBE_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c73f5dea/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 7d45fd6..f717d82 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -51,10 +51,12 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
+@Ignore
public class TestFuzzyRowFilterV2EndToEnd {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private final static byte fuzzyValue = (byte) 63;
@@ -75,8 +77,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt("hbase.client.scanner.caching", 1000);
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
- ConstantSizeRegionSplitPolicy.class.getName());
+ conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
// set no splits
conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
@@ -111,8 +112,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public void testEndToEnd() throws Exception {
String cf = "f";
- HTable ht =
- TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+ HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
// 10 byte row key - (2 bytes 4 bytes 4 bytes)
// 4 byte qualifier
@@ -250,8 +250,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
public void testFilterList() throws Exception {
String cf = "f";
String table = "TestFuzzyRowFiltersInFilterList";
- HTable ht =
- TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
+ HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
// 10 byte row key - (2 bytes 4 bytes 4 bytes)
// 4 byte qualifier
@@ -276,8 +275,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
p.setDurability(Durability.SKIP_WAL);
p.add(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
- LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
- + Bytes.toStringBinary(cq));
+ LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
}
}
}
@@ -320,8 +318,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
}
- private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2)
- throws IOException {
+ private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
String cf = "f";
Scan scan = new Scan();
scan.addFamily(cf.getBytes());
@@ -334,8 +331,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
long timeBeforeScan = System.currentTimeMillis();
while ((result = scanner.next()) != null) {
for (Cell kv : result.listCells()) {
- LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
- + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
+ LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
results.add(kv);
}
}
@@ -348,5 +344,3 @@ public class TestFuzzyRowFilterV2EndToEnd {
assertEquals(expectedSize, results.size());
}
}
-
-
[4/4] incubator-kylin git commit: KYLIN-984 Behavior change in
streaming data consuming
Posted by ma...@apache.org.
KYLIN-984 Behavior change in streaming data consuming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ab2abeec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ab2abeec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ab2abeec
Branch: refs/heads/0.8
Commit: ab2abeec50cc6f924d21da3f8a01dcf5a5c7c650
Parents: c73f5de
Author: honma <ho...@ebay.com>
Authored: Tue Sep 1 14:01:07 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 1 14:14:04 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/streaming/BootstrapConfig.java | 12 +++++++++++-
.../apache/kylin/job/streaming/StreamingBootstrap.java | 8 ++++++++
.../org/apache/kylin/job/streaming/StreamingCLI.java | 3 +++
.../java/org/apache/kylin/streaming/StreamFetcher.java | 8 ++------
4 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2abeec/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
index ceff3c1..029d4d2 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/BootstrapConfig.java
@@ -10,10 +10,20 @@ public class BootstrapConfig {
//one off default value set to true
private boolean oneOff = true;
private long start = 0L;
- private long end = 0L;
+ private long end = 0L;
+ private long margin = 0L;
+
private boolean fillGap;
+ public long getMargin() {
+ return margin;
+ }
+
+ public void setMargin(long margin) {
+ this.margin = margin;
+ }
+
public boolean isOneOff() {
return oneOff;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2abeec/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index 0811451..b7a8335 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -135,6 +135,14 @@ public class StreamingBootstrap {
final String streaming = bootstrapConfig.getStreaming();
Preconditions.checkNotNull(streaming, "streaming name cannot be empty");
final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(streaming);
+
+ if (bootstrapConfig.getMargin() != 0L) {
+ streamingConfig.setMargin(bootstrapConfig.getMargin());
+ logger.info("Margin is overwrite to " + streamingConfig.getMargin());
+ } else {
+ logger.info("Margin is default value: " + streamingConfig.getMargin());
+ }
+
Preconditions.checkArgument(streamingConfig != null, "cannot find kafka config:" + streaming);
if (!StringUtils.isEmpty(streamingConfig.getIiName())) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2abeec/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
index 78b1e93..8346ec0 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingCLI.java
@@ -79,6 +79,9 @@ public class StreamingCLI {
case "-fillGap":
bootstrapConfig.setFillGap(Boolean.parseBoolean(args[++i]));
break;
+ case "-margin":
+ bootstrapConfig.setMargin(Long.parseLong(args[++i]));
+ break;
default:
logger.warn("ignore this arg:" + argName);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ab2abeec/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
index f78389e..85d09be 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/StreamFetcher.java
@@ -59,14 +59,10 @@ public class StreamFetcher implements Callable<MicroStreamBatch> {
microStreamBatch = new MicroStreamBatch(partitionId);
clearCounter();
}
- StreamMessage streamMessage = peek(streamMessageQueue, 30000);
+ StreamMessage streamMessage = peek(streamMessageQueue, 60000);
if (streamMessage == null) {
logger.info("The stream queue is drained, current available stream count: " + microStreamBatch.size());
- if (!microStreamBatch.isEmpty()) {
- return microStreamBatch;
- } else {
- continue;
- }
+ return microStreamBatch;
}
if (streamMessage.getOffset() < 0) {
logger.warn("streaming encountered EOF, stop building");
[3/4] incubator-kylin git commit: KYLIN-973 add a tool to analyse
streaming logs
Posted by ma...@apache.org.
KYLIN-973 add a tool to analyse streaming logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b65d8c0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b65d8c0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b65d8c0b
Branch: refs/heads/0.8
Commit: b65d8c0b7aa2659d155c387c4a87049ce83d0c16
Parents: 08845a1
Author: honma <ho...@ebay.com>
Authored: Fri Aug 28 17:16:34 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 1 14:14:03 2015 +0800
----------------------------------------------------------------------
.../kylin/job/tools/StreamingLogsAnalyser.java | 96 ++++++++++++++++++++
1 file changed, 96 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b65d8c0b/job/src/main/java/org/apache/kylin/job/tools/StreamingLogsAnalyser.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/StreamingLogsAnalyser.java b/job/src/main/java/org/apache/kylin/job/tools/StreamingLogsAnalyser.java
new file mode 100644
index 0000000..9d83d01
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/tools/StreamingLogsAnalyser.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job.tools;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class StreamingLogsAnalyser {
+ public static void main(String[] args) {
+ int errorFileCount = 0;
+ List<Long> ellapsedTimes = Lists.newArrayList();
+
+ String patternStr = "(\\d{2}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})";
+ Pattern pattern = Pattern.compile(patternStr);
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+ format.setTimeZone(TimeZone.getTimeZone("GMT")); // NOTE: this must be GMT to calculate epoch date correctly
+
+ Preconditions.checkArgument(args.length == 1, "Usage: StreamingLogsAnalyser streaming_logs_folder");
+ for (File file : FileUtils.listFiles(new File(args[0]), new String[] { "log" }, false)) {
+ System.out.println("Processing file " + file.toString());
+
+ long startTime = 0;
+ long endTime = 0;
+ try {
+ List<String> contents = Files.readAllLines(file.toPath(), Charset.defaultCharset());
+ for (int i = 0; i < contents.size(); ++i) {
+ Matcher m = pattern.matcher(contents.get(i));
+ if (m.find()) {
+ startTime = format.parse("20" + m.group(1)).getTime();
+ break;
+ }
+ }
+
+ for (int i = contents.size() - 1; i >= 0; --i) {
+ Matcher m = pattern.matcher(contents.get(i));
+ if (m.find()) {
+ endTime = format.parse("20" + m.group(1)).getTime();
+ break;
+ }
+ }
+
+ if (startTime == 0 || endTime == 0) {
+ throw new RuntimeException("start time or end time is not found");
+ }
+
+ if (endTime - startTime < 60000) {
+ System.out.println("Warning: this job took less than one minute!!!! " + file.toString());
+ }
+
+ ellapsedTimes.add(endTime - startTime);
+
+ } catch (Exception e) {
+ System.out.println("Exception when processing log file " + file.toString());
+ System.out.println(e);
+ errorFileCount++;
+ }
+ }
+
+ System.out.println("Totally error files count " + errorFileCount);
+ System.out.println("Totally normal files processed " + ellapsedTimes.size());
+
+ long sum = 0;
+ for (Long x : ellapsedTimes) {
+ sum += x;
+ }
+ System.out.println("Avg build time " + (sum / ellapsedTimes.size()));
+ }
+}
[2/4] incubator-kylin git commit: KYLIN-977 adjust echcache settings
Posted by ma...@apache.org.
KYLIN-977 adjust echcache settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/42fa50d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/42fa50d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/42fa50d9
Branch: refs/heads/0.8
Commit: 42fa50d93049ccb6d0877b1961193d18226f48dc
Parents: b65d8c0
Author: honma <ho...@ebay.com>
Authored: Sat Aug 29 18:36:50 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 1 14:14:03 2015 +0800
----------------------------------------------------------------------
server/src/main/resources/ehcache.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/42fa50d9/server/src/main/resources/ehcache.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
index f8fc68d..29fce05 100644
--- a/server/src/main/resources/ehcache.xml
+++ b/server/src/main/resources/ehcache.xml
@@ -3,7 +3,7 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="500M"
+ maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>
@@ -11,7 +11,7 @@
eternal="false"
timeToIdleSeconds="86400"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="100M"
+ maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>
@@ -19,7 +19,7 @@
eternal="false"
timeToLiveSeconds="10800"
memoryStoreEvictionPolicy="LRU"
- maxBytesLocalHeap="100M"
+ maxBytesLocalHeap="50M"
>
<persistence strategy="none"/>
</cache>