You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/08/20 09:57:48 UTC
[01/17] incubator-kylin git commit: minor,
correct log message in AbstractHadoopJob
Repository: incubator-kylin
Updated Branches:
refs/heads/v1.0-release ff4c0771e -> f03dfedbf
minor, correct log message in AbstractHadoopJob
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b449590c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b449590c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b449590c
Branch: refs/heads/v1.0-release
Commit: b449590cc5c6aaa3eea9d19f922473bff6fce4b0
Parents: ff4c077
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Aug 14 14:33:00 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Aug 14 14:33:33 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b449590c/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 6d4267c..61a8e48 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -153,7 +153,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
Configuration jobConf = job.getConfiguration();
String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
[07/17] incubator-kylin git commit: Add blog “fast cubing”
Posted by lu...@apache.org.
Add blog “fast cubing”
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/265a528d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/265a528d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/265a528d
Branch: refs/heads/v1.0-release
Commit: 265a528d6ed5ce29b558fc3554753cf53049316a
Parents: 39bd19a
Author: shaofengshi <sh...@apache.org>
Authored: Mon Aug 17 14:38:04 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Aug 17 14:38:34 2015 +0800
----------------------------------------------------------------------
website/_dev/new_metadata.md | 2 +-
website/_dev/test_minicluster.md | 2 +-
website/_dev/web_tech.md | 2 +-
website/_docs/howto/howto_jdbc.md | 2 +-
website/_docs/howto/howto_use_restapi_in_js.md | 2 +-
website/_docs/install/advance_settings.md | 2 +-
website/_posts/blog/2015-08-15-fast-cubing.md | 118 ++++++++++++++++++++
website/images/blog/by-layer-cubing.png | Bin 0 -> 130733 bytes
website/images/blog/by-segment-cubing.png | Bin 0 -> 80771 bytes
website/images/blog/cube-spanning-tree.png | Bin 0 -> 27581 bytes
10 files changed, 124 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_dev/new_metadata.md
----------------------------------------------------------------------
diff --git a/website/_dev/new_metadata.md b/website/_dev/new_metadata.md
index 851ab85..d52aa61 100644
--- a/website/_dev/new_metadata.md
+++ b/website/_dev/new_metadata.md
@@ -1,6 +1,6 @@
---
layout: dev
-title: "New meta data model structure"
+title: "New Metadata Model"
categories: development
permalink: /development/new_metadata.html
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_dev/test_minicluster.md
----------------------------------------------------------------------
diff --git a/website/_dev/test_minicluster.md b/website/_dev/test_minicluster.md
index 44d998d..f0b21ef 100644
--- a/website/_dev/test_minicluster.md
+++ b/website/_dev/test_minicluster.md
@@ -1,6 +1,6 @@
---
layout: dev
-title: "Run Kylin test case with HBase Mini Cluster"
+title: "Run Kylin Test Case with Mini Cluster"
categories: development
permalink: /development/test_minicluster.html
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_dev/web_tech.md
----------------------------------------------------------------------
diff --git a/website/_dev/web_tech.md b/website/_dev/web_tech.md
index 90ffe76..0603eda 100644
--- a/website/_dev/web_tech.md
+++ b/website/_dev/web_tech.md
@@ -1,6 +1,6 @@
---
layout: dev
-title: "Kylin web front end summary"
+title: "Kylin Web Summary"
categories: development
permalink: /development/web_tech.html
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_docs/howto/howto_jdbc.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_jdbc.md b/website/_docs/howto/howto_jdbc.md
index df79153..45a2f4e 100644
--- a/website/_docs/howto/howto_jdbc.md
+++ b/website/_docs/howto/howto_jdbc.md
@@ -1,6 +1,6 @@
---
layout: docs
-title: How to use kylin remote jdbc driver
+title: How to Use kylin Remote JDBC Driver
categories: howto
permalink: /docs/howto/howto_jdbc.html
version: v0.7.2
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_docs/howto/howto_use_restapi_in_js.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_use_restapi_in_js.md b/website/_docs/howto/howto_use_restapi_in_js.md
index 1b2c78b..54b2c3e 100644
--- a/website/_docs/howto/howto_use_restapi_in_js.md
+++ b/website/_docs/howto/howto_use_restapi_in_js.md
@@ -1,6 +1,6 @@
---
layout: docs
-title: How to use Restful API in Javascript
+title: How to Use Restful API in Javascript
categories: howto
permalink: /docs/howto/howto_use_restapi_in_js.html
version: v0.8
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_docs/install/advance_settings.md
----------------------------------------------------------------------
diff --git a/website/_docs/install/advance_settings.md b/website/_docs/install/advance_settings.md
index 43b0bea..3630954 100644
--- a/website/_docs/install/advance_settings.md
+++ b/website/_docs/install/advance_settings.md
@@ -1,6 +1,6 @@
---
layout: docs
-title: "Advance settings of Kylin environment"
+title: "Advance Settings of Kylin Environment"
categories: install
permalink: /docs/install/advance_settings.html
version: v0.7.2
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/_posts/blog/2015-08-15-fast-cubing.md
----------------------------------------------------------------------
diff --git a/website/_posts/blog/2015-08-15-fast-cubing.md b/website/_posts/blog/2015-08-15-fast-cubing.md
new file mode 100644
index 0000000..2d4ae81
--- /dev/null
+++ b/website/_posts/blog/2015-08-15-fast-cubing.md
@@ -0,0 +1,118 @@
+---
+layout: post-blog
+title: "Fast Cubing Algorithm in Apache Kylin: Concept"
+date: 2015-08-15 12:28:00
+author: Shaofeng Shi
+categories: blog
+---
+
+This article is the first post of a series document that introducing the fast cubing algorithm in Apache Kylin; The new algorithm will be released in the near future (v1.1 or later); Now it is under internal testing and pilot; All the source code has been published in Kylin's Git repository <https://github.com/apache/incubator-kylin/> on 0.8 branch.
+
+
+## By Layer Cubing Algorithm
+Before introduce the fast cubing, I'd like to make a brief introduction on the as-is cubing algorithm, which is called "by layer cubing";
+
+As its name indicates, a full cube is calculated by layer: N-dimension, N-1 dimension, N-2 dimension, ... until 0 dimension; Each layer's calculation is based on its parent layer (except the first, which base on source data); So this algorithm need N rounds of MapReduce running in sequence;
+
+In the MapReduce, the key is the composite of the dimensions, the value is the composite of the measures; When the mapper reads a key-value pair, it calculates its possible child cuboids; for each child cuboid, remove 1 dimension from the key, and then output the new key and value to the reducer;
+
+The reducer gets the values grouped by key; it aggregates the measures, and then output to HDFS; one layer's MR is finished;
+
+When all layers are finished, the cube is calculated;
+
+Figure 1 describes the flow:
+
+
+Figure 1: By layer cubing
+
+
+**Advantage**
+
+* This algorithm is clear as it mostly leverage the capability of Hadoop; The sorting, grouping and shuffling are all taken care by Hadoop, developer just focus on the cubing logic;
+* Benefiting from Hadoop's maturity, this algorithm is very stable; In our experience there is seldom case that the mapper or reducer could fail; Even if your Hadoop cluster is small or is busy, it can finish in the end;
+
+
+**Disadvantage**
+
+* If the cube has many dimensions, it need the same number of MR jobs; As each Hadoop job scheduling need extra resource, the overhead cost to Hadoop is considerable;
+* This algorithm causes too much shuffling to Hadoop; The mapper doesn't do aggregation, all the records that having same dimension values in next layer will be omitted to Hadoop, and then sent to reducer for aggregation;
+* Many writes to HDFS: each layer's cubing need write its output to HDFS for next layer MR to consume; In the end, Kylin need another round MR to convert these output files to HBase HFile for bulk load; These jobs generates many intermediate files in HDFS;
+* All in all: the performance is not good, especially when the cube has many dimensions (>10);
+
+
+
+## Fast Cubing Algorithm
+The fast cubing algorithm is also called "by segment cubing"; The core idea is, each mapper calculates the feed data block into a small cube segment (with all cuboids), and then output all key/values to reducer; The reducer aggregates them into one big cube segment, finishing the cubing; Figure 2 illustrates the flow;
+
+
+
+Figure 2: By segment cubing
+
+### Pre-aggregation in Mapper
+Compared with the origin algorithm, the fast cubing has two main differences:
+
+* The mapper will do pre-aggregation, this will reduce the number of records that the mapper output to Hadoop, and also reduce the number that reducer need to aggregate;
+* One round MR can calculate all cuboids;
+
+Let take an example: a cube has 4 dimensions: A, B, C, D; Each mapper has 1 million source records to process; The column cardinality in the mapper is Car(A), Car(B), Car(C) and Car(D);
+
+* When aggregate the source records to base cuboid (1111), with the old "by layer" algorithm, the mapper will output 1 million records to Hadoop; With the fast cubing algorithm, after the pre-aggregation, it only outputs the number of [distinct A, B, C, D] records to Hadoop, which is certainly smaller than source data; In a normal case, it can be 1/10 to 1/1000 of the source records size;
+
+* When aggregate from parent to a child cuboid, say from base cuboid (1111) to 3-dimension cuboid 0111, the dimension A will be aggregated; We assume the dimension A is independent with other dimensions, after aggregation, the cuboid 0111's size will be about 1/Card(A) of the base cuboid; So the output will be reduced to 1/Card(A) of the original one in this step;
+
+Totally, assume the average cardinality of the dimensions is Card(N), the records that written from mapper to reducer can be reduced to 1/Card(N) of origin size; The less output to Hadoop, the less I/O and computing, the better performance;
+
+### Cuboid spanning tree
+Inner the mapper, there is another change in the cuboid spanning tree visiting order; In origin cubing, Kylin calculates the cuboids with Broad First Search order; In the fast cubing, it uses Depth First Search order, to reduce the cuboids that need be cached in memory;
+
+Let's look at the figure 3, which is a full spanning tree of a 4-dimension cube: before 0-Dimenson cuboid "\*" be aggregated, cuboid "ABCD", "BCD", "CD" and "D" will be cached in memory; As "\*" has no child, it will be outputted once be calculated, then comes "D"; After "C" be outputted, "CD" will be outputted as all of its children has finished; The memory for a cuboid can be released once it be outputted; The base cuboid ABCD will be the last one to finish;
+
+
+Figure 3: Cuboid spanning tree
+
+With the DFS visiting order, the output from a mapper is almost fully sorted, as the cuboid ID is at the beginning position of row key, and inner a cuboid the rows are already sorted:
+
+```
+0000
+0001[D0]
+0001[D1]
+....
+0010[C0]
+0010[C1]
+....
+0011[C0][D0]
+0011[C0][D1]
+....
+....
+1111[A0][B0][C0][D0]
+....
+```
+Since the outputs from mapper are already sorted, Hadoop's shuffle and sort would be faster;
+
+Besides, mapper's pre-aggregation happens in memory, this avoids unnecessary disk and network I/O, and the overhead to Hadoop is considerable reduced;
+
+###OutOfMemory error
+During the development phase, we encountered the OutOfMemory error in mappers; this could happen when:
+
+ a) The mapper's JVM heap size is small;
+ b) "Distinct count" measure is used (HyperLogLog is space consuming)
+ c) The spanning tree is too deep (too many dimensions);
+
+We realized that Kylin couldn't assume the mappers always have enough memory; The cubing algorithm need be self-adapting to various situations; A lot of efforts were put on optimizing the memory use and spilling data to disk when proactively detects an OutOfMemory error; The result is promising, the OOM error is rare to occur now;
+
+Here let's do a summary on the fast cubing algorithm;
+
+**Advantage**
+
+* It is faster than the old method; can reduce 30% to 50% overall building time from our comparison test;
+* It produces less work load on Hadoop, and leaves less intermediate files on HDFS;
+* The cubing code can be easily reused by other cubing engines like Streaming and Spark;
+
+**Disadvantage**
+
+* The algorithm is a little complicated; This adds the maintenance effort;
+* Although the algorithm can spill data to disk automatically, it still wish the mapper has enough memory to gain best performance; User need more knowledge to tune the cubing; (this can be discussed in detail later)
+
+## Other Enhancements in Fast Cubing
+
+Together with the fast cubing algorithm, we also introduced several other enhancements in the cube build flow; I's like to elaborate in next articles, please keep posted here; If you have interest on Apache Kylin, welcome to visit our home page <http://kylin.incubator.apache.org/> and subscribe our development mailing list at <de...@kylin.incubator.apache.org>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/images/blog/by-layer-cubing.png
----------------------------------------------------------------------
diff --git a/website/images/blog/by-layer-cubing.png b/website/images/blog/by-layer-cubing.png
new file mode 100644
index 0000000..b5238d2
Binary files /dev/null and b/website/images/blog/by-layer-cubing.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/images/blog/by-segment-cubing.png
----------------------------------------------------------------------
diff --git a/website/images/blog/by-segment-cubing.png b/website/images/blog/by-segment-cubing.png
new file mode 100644
index 0000000..02fbec3
Binary files /dev/null and b/website/images/blog/by-segment-cubing.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/265a528d/website/images/blog/cube-spanning-tree.png
----------------------------------------------------------------------
diff --git a/website/images/blog/cube-spanning-tree.png b/website/images/blog/cube-spanning-tree.png
new file mode 100644
index 0000000..71c464c
Binary files /dev/null and b/website/images/blog/cube-spanning-tree.png differ
[11/17] incubator-kylin git commit: KYLIN-771
Posted by lu...@apache.org.
KYLIN-771
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cc4fdf12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cc4fdf12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cc4fdf12
Branch: refs/heads/v1.0-release
Commit: cc4fdf12af158dbc9d110cdbfa7f1b19a2e04bdc
Parents: 7a3941d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Aug 19 15:57:25 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Aug 19 15:57:25 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/CacheService.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cc4fdf12/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index fd919d6..ac17728 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -31,8 +31,11 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.rest.controller.QueryController;
import org.apache.kylin.storage.hybrid.HybridManager;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Caching;
import org.springframework.stereotype.Component;
/**
@@ -44,6 +47,7 @@ public class CacheService extends BasicService {
@Autowired
private CubeService cubeService;
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public void rebuildCache(Broadcaster.TYPE cacheType, String cacheKey) {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
@@ -106,6 +110,7 @@ public class CacheService extends BasicService {
}
}
+ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
public void removeCache(Broadcaster.TYPE cacheType, String cacheKey) {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
@@ -116,7 +121,7 @@ public class CacheService extends BasicService {
case CUBE_DESC:
getCubeDescManager().removeLocalCubeDesc(cacheKey);
break;
- case PROJECT:
+ case PROJECT:
ProjectManager.clearCache();
break;
case INVERTED_INDEX:
[10/17] incubator-kylin git commit: KYLIN-926 clean all runtime files
Posted by lu...@apache.org.
KYLIN-926 clean all runtime files
KYLIN-926
KYLIN-926
KYLIN-926
KYLIN-926
KYLIN-926 clean parent file
KYLIN-926 cond
KYLIN-926
KYLIN-926 fix /tmp/cardinality and minicluster tgz
KYLIN-926
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7a3941d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7a3941d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7a3941d8
Branch: refs/heads/v1.0-release
Commit: 7a3941d81c2f7df036c7d9f0a96045055730433f
Parents: 2beba36
Author: honma <ho...@ebay.com>
Authored: Mon Aug 17 16:32:10 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 19 13:14:32 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 15 +-
.../apache/kylin/common/util/HbaseImporter.java | 9 +-
.../kylin/job/constant/BatchConstants.java | 33 ++--
.../kylin/job/hadoop/AbstractHadoopJob.java | 45 +++--
.../cardinality/HiveColumnCardinalityJob.java | 3 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 10 +-
.../kylin/job/hadoop/cube/NDCuboidMapper.java | 4 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +-
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 4 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 1 +
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../org/apache/kylin/job/ExportHBaseData.java | 41 +++--
.../org/apache/kylin/job/ImportHBaseData.java | 174 -------------------
.../job/hadoop/cube/MergeCuboidJobTest.java | 8 +-
.../apache/kylin/query/test/IIQueryTest.java | 2 -
.../apache/kylin/rest/service/CubeService.java | 3 -
16 files changed, 84 insertions(+), 272 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0c7b813..0e4cc07 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -52,15 +52,6 @@ public class KylinConfig {
public static final String KYLIN_REST_SERVERS = "kylin.rest.servers";
public static final String KYLIN_REST_TIMEZONE = "kylin.rest.timezone";
- /**
- * The dir containing scripts for kylin. For example: /usr/lib/kylin/bin
- */
- public static final String KYLIN_SCRIPT_DIR = "kylin.script.dir";
- /**
- * The script file name for generating table metadat from hive. For example:
- * generateTable.sh
- */
- public static final String KYLIN_SCRIPT_GEN_TABLE_META = "kylin.script.genTableMeta";
public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
@@ -124,6 +115,7 @@ public class KylinConfig {
public static final String MAIL_SENDER = "mail.sender";
public static final String KYLIN_HOME = "KYLIN_HOME";
+
public static final String KYLIN_CONF = "KYLIN_CONF";
private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class);
@@ -233,8 +225,7 @@ public class KylinConfig {
/**
* Find config from environment. The Search process: 1. Check the
- * $KYLIN_CONF/kylin.properties 2. Check the /etc/kylin/kylin.properties 3.
- * Check the kylin.properties in classpath
+ * $KYLIN_CONF/kylin.properties 2. Check the $KYLIN_HOME/conf/kylin.properties
*
* @return
*/
@@ -284,7 +275,7 @@ public class KylinConfig {
if (!root.endsWith("/")) {
root += "/";
}
- return root + getMetadataUrlPrefix();
+ return root + getMetadataUrlPrefix() + "/";
}
public String getKylinJobLogDir() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
index 0465be0..097272f 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HbaseImporter.java
@@ -35,9 +35,6 @@ import org.apache.kylin.common.persistence.HBaseConnection;
import com.google.common.base.Preconditions;
-/**
- * Created by Hongbin Ma(Binmahone) on 2/6/15.
- */
public class HbaseImporter {
private static final Log logger = LogFactory.getLog(HbaseImporter.class);
@@ -56,12 +53,12 @@ public class HbaseImporter {
return;
}
- File folder = new File("/tmp/hbase-export/");
+ File folder = File.createTempFile("hbase-import", "tmp");
if (folder.exists()) {
- FileUtils.deleteDirectory(folder);
+ FileUtils.forceDelete(folder);
}
folder.mkdirs();
- folder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(folder);
//TarGZUtil.uncompressTarGZ(exportFile, folder);
FileUtil.unTar(exportFile, folder);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
index 0ac04fb..34c5760 100644
--- a/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
+++ b/job/src/main/java/org/apache/kylin/job/constant/BatchConstants.java
@@ -24,29 +24,30 @@ package org.apache.kylin.job.constant;
*/
public interface BatchConstants {
- public static final char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
+ char INTERMEDIATE_TABLE_ROW_DELIMITER = 127;
- public static final String CFG_CUBE_NAME = "cube.name";
- public static final String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
+ String CFG_CUBE_NAME = "cube.name";
+ String CFG_CUBE_SEGMENT_NAME = "cube.segment.name";
- public static final String CFG_II_NAME = "ii.name";
- public static final String CFG_II_SEGMENT_NAME = "ii.segment.name";
+ String CFG_II_NAME = "ii.name";
+ String CFG_II_SEGMENT_NAME = "ii.segment.name";
- public static final String INPUT_DELIM = "input.delim";
+ String OUTPUT_PATH = "output.path";
- public static final String OUTPUT_PATH = "output.path";
+ String TABLE_NAME = "table.name";
+ String TABLE_COLUMNS = "table.columns";
- public static final String TABLE_NAME = "table.name";
- public static final String TABLE_COLUMNS = "table.columns";
+ String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
- public static final String CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER = "cube.intermediate.table.row.delimiter";
+ String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder";
- public static final String MAPREDUCE_COUTNER_GROUP_NAME = "Cube Builder";
+ String MAPPER_SAMPLE_NUMBER = "mapper.sample.number";
+ String REGION_NUMBER = "region.number";
+ String CUBE_CAPACITY = "cube.capacity";
- public static final String MAPPER_SAMPLE_NUMBER = "mapper.sample.number";
- public static final String REGION_NUMBER = "region.number";
- public static final String CUBE_CAPACITY = "cube.capacity";
+ String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/";
+ String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/";
- public static final int COUNTER_MAX = 100000;
- public static final int ERROR_RECORD_THRESHOLD = 100;
+ int COUNTER_MAX = 100000;
+ int ERROR_RECORD_THRESHOLD = 100;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 61a8e48..4c60c52 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -36,6 +36,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
@@ -48,7 +49,6 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -82,11 +82,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected static final Option OPTION_NCUBOID_LEVEL = OptionBuilder.withArgName("level").hasArg().isRequired(true).withDescription("N-Cuboid build level, e.g. 1, 2, 3...").create("level");
protected static final Option OPTION_PARTITION_FILE_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Partition file path.").create("input");
protected static final Option OPTION_HTABLE_NAME = OptionBuilder.withArgName("htable name").hasArg().isRequired(true).withDescription("HTable name").create("htablename");
- protected static final Option OPTION_KEY_COLUMN_PERCENTAGE = OptionBuilder.withArgName("rowkey column percentage").hasArg().isRequired(true).withDescription("Percentage of row key columns").create("columnpercentage");
- protected static final Option OPTION_KEY_SPLIT_NUMBER = OptionBuilder.withArgName("key split number").hasArg().isRequired(true).withDescription("Number of key split range").create("splitnumber");
protected String name;
- protected String description;
protected boolean isAsync = false;
protected OptionsHelper optionsHelper = new OptionsHelper();
@@ -129,16 +126,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return retVal;
}
- protected static void runJob(Tool job, String[] args) {
- try {
- int exitCode = ToolRunner.run(job, args);
- System.exit(exitCode);
- } catch (Exception e) {
- e.printStackTrace(System.err);
- System.exit(5);
- }
- }
-
private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
protected void setJobClasspath(Job job) {
@@ -222,7 +209,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected void attachKylinPropsAndMetadata(CubeInstance cube, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
- tmp.delete(); // we need a directory, so delete the file first
+ FileUtils.forceDelete(tmp);
File metaDir = new File(tmp, "meta");
metaDir.mkdirs();
@@ -249,29 +236,41 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
dumpResources(kylinConfig, metaDir, dumpList);
// hadoop distributed cache
- String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
-
+
}
protected void cleanupTempConfFile(Configuration conf) {
String tempMetaFileString = conf.get("tmpfiles");
+ logger.info("tempMetaFileString is : " + tempMetaFileString);
if (tempMetaFileString != null) {
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- tempMetaFile.getParentFile().delete();
+ if (tempMetaFileString.startsWith("file://")) {
+ tempMetaFileString = tempMetaFileString.substring("file://".length());
+ File tempMetaFile = new File(tempMetaFileString);
+ if (tempMetaFile.exists()) {
+ try {
+ FileUtils.forceDelete(tempMetaFile.getParentFile());
+
+ } catch (IOException e) {
+ logger.warn("error when deleting " + tempMetaFile, e);
+ }
+ } else {
+ logger.info("" + tempMetaFileString + " does not exist");
+ }
+ } else {
+ logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
}
}
}
protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
File tmp = File.createTempFile("kylin_job_meta", "");
- tmp.delete(); // we need a directory, so delete the file first
+ FileUtils.forceDelete(tmp);
File metaDir = new File(tmp, "meta");
metaDir.mkdirs();
- metaDir.getParentFile().deleteOnExit();
// write kylin.properties
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -296,7 +295,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
dumpResources(kylinConfig, metaDir, dumpList);
// hadoop distributed cache
- String hdfsMetaDir = "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ String hdfsMetaDir = "file://" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
logger.info("HDFS meta dir is: " + hdfsMetaDir);
conf.set("tmpfiles", hdfsMetaDir);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
index e47e05a..ed0abe7 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
/**
@@ -44,7 +45,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
@SuppressWarnings("static-access")
protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table");
- public static final String OUTPUT_PATH = "/tmp/cardinality";
+ public static final String OUTPUT_PATH = BatchConstants.CFG_KYLIN_HDFS_TEMP_DIR + "cardinality";
public HiveColumnCardinalityJob() {
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 6e2b467..e13e011 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,29 +18,28 @@
package org.apache.kylin.job.hadoop.cube;
+import java.io.IOException;
+
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* @author yangli9
*/
@@ -88,8 +87,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
printUsage(options);
throw e;
} finally {
- if (job != null)
+ if (job != null) {
cleanupTempConfFile(job.getConfiguration());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
index 7a69c91..f846232 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NDCuboidMapper.java
@@ -115,7 +115,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Skipped records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L);
skipCounter++;
if (skipCounter % BatchConstants.COUNTER_MAX == 0) {
logger.info("Skipped " + skipCounter + " records!");
@@ -123,7 +123,7 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
return;
}
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Processed records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L);
handleCounter++;
if (handleCounter % BatchConstants.COUNTER_MAX == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
index b6e4233..e5c891b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/NewBaseCuboidMapper.java
@@ -339,7 +339,7 @@ public class NewBaseCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Text, Text, T
} catch (Throwable t) {
logger.error("", t);
- context.getCounter(BatchConstants.MAPREDUCE_COUTNER_GROUP_NAME, "Error records").increment(1L);
+ context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Error records").increment(1L);
return;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
index e9e30c2..1350077 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/hbase/CreateHTableJob.java
@@ -53,10 +53,6 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * @author George Song (ysong1)
- */
-
public class CreateHTableJob extends AbstractHadoopJob {
protected static final Logger logger = LoggerFactory.getLogger(CreateHTableJob.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index d39a27e..3916c40 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -237,6 +237,7 @@ public class BuildIIWithEngineTest {
private void exportHBaseData() throws IOException {
ExportHBaseData export = new ExportHBaseData();
export.exportTables();
+ export.tearDown();
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index d1c9745..41399c7 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -24,6 +24,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -43,7 +44,6 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.apache.tools.ant.filters.StringInputStream;
-import org.codehaus.plexus.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
index 9e302b9..557dd39 100644
--- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
+++ b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
@@ -21,6 +21,7 @@ package org.apache.kylin.job;
import java.io.File;
import java.io.IOException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -31,7 +32,7 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HBaseMiniclusterHelper;
import org.apache.kylin.common.util.SSHClient;
-import org.codehaus.plexus.util.FileUtils;
+import org.apache.kylin.job.constant.BatchConstants;
public class ExportHBaseData {
@@ -39,10 +40,13 @@ public class ExportHBaseData {
HTableDescriptor[] allTables;
Configuration config;
HBaseAdmin hbase;
- CliCommandExecutor cli = null;
- String exportFolder;
- String backupArchive = null;
+ CliCommandExecutor cli;
+ String exportHdfsFolder;
+ String exportLocalFolderParent;
+ String exportLocalFolder;
+ String backupArchive;
String tableNameBase;
+ long currentTIME;
public ExportHBaseData() {
try {
@@ -53,9 +57,6 @@ public class ExportHBaseData {
}
private void setup() throws IOException {
- long currentTIME = System.currentTimeMillis();
- exportFolder = "/tmp/hbase-export/" + currentTIME + "/";
- backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz";
KylinConfig.destoryInstance();
System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
@@ -63,6 +64,12 @@ public class ExportHBaseData {
kylinConfig = KylinConfig.getInstanceFromEnv();
cli = kylinConfig.getCliCommandExecutor();
+ currentTIME = System.currentTimeMillis();
+ exportHdfsFolder = kylinConfig.getHdfsWorkingDirectory() + "hbase-export/" + currentTIME + "/";
+ exportLocalFolderParent = BatchConstants.CFG_KYLIN_LOCAL_TEMP_DIR + "hbase-export/";
+ exportLocalFolder = exportLocalFolderParent + currentTIME + "/";
+ backupArchive = exportLocalFolderParent + "hbase-export-at-" + currentTIME + ".tar.gz";
+
String metadataUrl = kylinConfig.getMetadataUrl();
// split TABLE@HBASE_URL
int cut = metadataUrl.indexOf('@');
@@ -84,16 +91,16 @@ public class ExportHBaseData {
// cleanup hdfs
try {
- if (cli != null && exportFolder != null) {
- cli.execute("hadoop fs -rm -r " + exportFolder);
+ if (cli != null && exportHdfsFolder != null) {
+ cli.execute("hadoop fs -rm -r " + exportHdfsFolder);
}
} catch (IOException e) {
e.printStackTrace();
}
// cleanup sandbox disk
try {
- if (cli != null && exportFolder != null) {
- cli.execute("rm -r " + exportFolder);
+ if (cli != null && exportLocalFolder != null) {
+ cli.execute("rm -r " + exportLocalFolder);
}
} catch (IOException e) {
e.printStackTrace();
@@ -107,24 +114,22 @@ public class ExportHBaseData {
} catch (IOException e) {
e.printStackTrace();
}
-
- KylinConfig.destoryInstance();
-
}
public void exportTables() throws IOException {
- cli.execute("mkdir -p " + exportFolder);
+ cli.execute("mkdir -p " + exportLocalFolderParent);
for (HTableDescriptor table : allTables) {
String tName = table.getNameAsString();
if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
continue;
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportFolder + tName);
+ cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportHdfsFolder + tName);
}
- cli.execute("hadoop fs -copyToLocal " + exportFolder + " " + exportFolder);
- cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportFolder + " .");
+ cli.execute("hadoop fs -copyToLocal " + exportHdfsFolder + " " + exportLocalFolderParent);
+ cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportLocalFolderParent + " " + currentTIME);
+
downloadToLocal();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
deleted file mode 100644
index 60983b6..0000000
--- a/job/src/test/java/org/apache/kylin/job/ImportHBaseData.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
-import org.apache.kylin.common.persistence.HBaseResourceStore;
-import org.apache.kylin.common.util.AbstractKylinTestCase;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HBaseMiniclusterHelper;
-import org.apache.kylin.common.util.SSHClient;
-import org.apache.kylin.common.util.TarGZUtil;
-
-public class ImportHBaseData {
-
- KylinConfig kylinConfig;
- HTableDescriptor[] allTables;
- Configuration config;
- HBaseAdmin hbase;
- CliCommandExecutor cli = null;
- String importFolder = "/tmp/hbase-export/";
- String backupArchive = null;
- String tableNameBase;
-
- public void setup() throws IOException {
-
- KylinConfig.destoryInstance();
- System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- cli = kylinConfig.getCliCommandExecutor();
-
- String metadataUrl = kylinConfig.getMetadataUrl();
- // split TABLE@HBASE_URL
- int cut = metadataUrl.indexOf('@');
- tableNameBase = metadataUrl.substring(0, cut);
- String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
-
- HConnection conn = HBaseConnection.get(hbaseUrl);
- try {
- hbase = new HBaseAdmin(conn);
- config = hbase.getConfiguration();
- //allTables = hbase.listTables();
- } catch (IOException e) {
- e.printStackTrace();
- throw e;
- }
-
- uploadTarballToRemote();
- }
-
- public void tearDown() {
- // cleanup sandbox disk
- try {
- if (cli != null && importFolder != null) {
- cli.execute("rm -r " + importFolder);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- KylinConfig.destoryInstance();
-
- }
-
- public void importTables() throws IOException {
- // create the metadata htables
- new HBaseResourceStore(KylinConfig.getInstanceFromEnv());
-
- List<String> tablelocations = getTablesBackupLocations(importFolder);
- for (String tableLocation : tablelocations) {
- String table = tableLocation.substring(tableLocation.lastIndexOf("/") + 1);
-
- if (!(table.equalsIgnoreCase(tableNameBase) || table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))) {
- continue;
- }
-
- if (table.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX)) {
- // create the cube table; otherwise the import will fail.
- HBaseConnection.createHTableIfNeeded(KylinConfig.getInstanceFromEnv().getStorageUrl(), table, "F1", "F2");
- }
- cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Import " + table + " file://" + tableLocation);
- }
-
- }
-
- public void uploadTarballToRemote() throws IOException {
-
- cli.execute("mkdir -p /tmp/hbase-export/");
- @SuppressWarnings("unused")
- SSHClient ssh = new SSHClient(kylinConfig.getRemoteHadoopCliHostname(), kylinConfig.getRemoteHadoopCliUsername(), kylinConfig.getRemoteHadoopCliPassword());
- try {
- // ssh.scpFileToRemote("../examples/test_case_data/minicluster/hbase-export.tar.gz", importFolder);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- cli.execute("tar -xzf /tmp/hbase-export/hbase-export.tar.gz --directory=" + importFolder);
- }
-
- private List<String> getTablesBackupLocations(String exportBase) throws IOException {
- File exportFile = new File("../examples/test_case_data/minicluster/hbase-export.tar.gz");
-
- if (!exportFile.exists()) {
- return null;
- }
-
- File folder = new File("/tmp/hbase-export/");
-
- if (folder.exists()) {
- folder.delete();
- }
-
- folder.mkdirs();
- folder.deleteOnExit();
-
- TarGZUtil.uncompressTarGZ(exportFile, folder);
-
- String[] child = folder.list();
-
- assert child.length == 1;
-
- String backupTime = child[0];
-
- File backupFolder = new File(folder, backupTime);
-
- String[] tableNames = backupFolder.list();
-
- List<String> locations = new ArrayList<String>(15);
-
- for (String t : tableNames) {
- locations.add(exportBase + backupTime + "/" + t);
- }
-
- return locations;
- }
-
- public static void main(String[] args) {
- ImportHBaseData export = new ImportHBaseData();
- try {
- export.setup();
- export.importTables();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- //export.tearDown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
index f9d4f45..72f515a 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJobTest.java
@@ -60,16 +60,16 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase {
String jobname = "merge_cuboid";
File baseFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "base");
- baseFolder.delete();
+ FileUtils.forceDelete(baseFolder);
baseFolder.mkdir();
FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), baseFolder);
- baseFolder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(baseFolder);
File eightFolder = File.createTempFile("kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa-", "8d");
- eightFolder.delete();
+ FileUtils.forceDelete(eightFolder);
eightFolder.mkdir();
FileUtils.copyDirectory(new File("src/test/resources/data/base_cuboid"), eightFolder);
- eightFolder.deleteOnExit();
+ FileUtils.forceDeleteOnExit(eightFolder);
FileUtil.fullyDelete(new File(output));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
index 198fb2f..80ff4bb 100644
--- a/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/IIQueryTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query.test;
-import java.io.File;
import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -59,5 +58,4 @@ public class IIQueryTest extends KylinQueryTest {
execAndCompQuery("src/test/resources/query/sql_ii", null, true);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7a3941d8/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index bd81c45..c3e6694 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -313,7 +313,6 @@ public class CubeService extends BasicService {
* @param cube
* @return
* @throws IOException
- * @throws CubeIntegrityException
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@@ -339,7 +338,6 @@ public class CubeService extends BasicService {
* Update a cube status from ready to disabled.
*
* @return
- * @throws CubeIntegrityException
* @throws IOException
* @throws JobException
*/
@@ -528,7 +526,6 @@ public class CubeService extends BasicService {
*
* @throws IOException
* @throws JobException
- * @throws CubeIntegrityException
*/
private void releaseAllSegments(CubeInstance cube) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null);
[09/17] incubator-kylin git commit: KYLIN-952 User can trigger a
Refresh job on an non-existing cube segment via REST API
Posted by lu...@apache.org.
KYLIN-952 User can trigger a Refresh job on an non-existing cube segment via REST API
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2beba36e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2beba36e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2beba36e
Branch: refs/heads/v1.0-release
Commit: 2beba36ea9886a7ca45ac490c5340ba5729da7ad
Parents: 4ab6372
Author: shaofengshi <sh...@apache.org>
Authored: Tue Aug 18 10:06:01 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Aug 18 10:06:39 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/JobService.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2beba36e/server/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
index a0be497..ff61cc3 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -154,6 +154,19 @@ public class JobService extends BasicService {
CubeSegment newSeg = getCubeManager().mergeSegments(cube, startDate, endDate, forceMergeEmptySeg);
job = builder.mergeJob(newSeg);
} else if (buildType == CubeBuildTypeEnum.REFRESH) {
+ List<CubeSegment> readySegs = cube.getSegment(SegmentStatusEnum.READY);
+ boolean segExists = false;
+ for (CubeSegment aSeg : readySegs) {
+ if (aSeg.getDateRangeStart() == startDate && aSeg.getDateRangeEnd() == endDate) {
+ segExists = true;
+ break;
+ }
+ }
+
+ if (segExists == false) {
+ throw new IllegalArgumentException("You can only refresh an existing segment, but there is no ready segment for start time:" + startDate + ", end time: " + endDate);
+ }
+
CubeSegment refreshSeg = getCubeManager().refreshSegment(cube, startDate, endDate);
job = builder.buildJob(refreshSeg);
} else {
[15/17] incubator-kylin git commit: minor,
less verbose HyperLogLogCounterTest
Posted by lu...@apache.org.
minor, less verbose HyperLogLogCounterTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/19e4c236
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/19e4c236
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/19e4c236
Branch: refs/heads/v1.0-release
Commit: 19e4c2363ba88a9713c9cfa56f9c1b2c93098692
Parents: ebfa38d
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Aug 20 14:29:01 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Aug 20 14:29:01 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/HyperLogLogCounterTest.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/19e4c236/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index 571a12d..5943bb6 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -119,16 +119,15 @@ public class HyperLogLogCounterTest {
double absError = 0;
int n = 100;
for (int i = 0; i < n; i++) {
- System.out.println("============" + i);
double e = merge();
error += e;
absError += Math.abs(e);
}
System.out.println("Total average error is " + error / n + " and absolute error is " + absError / n);
- System.out.println("errorCount1 is " + errorCount1 + "!");
- System.out.println("errorCount2 is " + errorCount2 + "!");
- System.out.println("errorCount3 is " + errorCount3 + "!");
+ System.out.println(" errorRateCount1 is " + errorCount1 + "!");
+ System.out.println(" errorRateCount2 is " + errorCount2 + "!");
+ System.out.println(" errorRateCount3 is " + errorCount3 + "!");
Assert.assertTrue(errorCount1 <= n * 0.40);
Assert.assertTrue(errorCount2 <= n * 0.08);
@@ -161,9 +160,8 @@ public class HyperLogLogCounterTest {
long estimate = mergeHllc.getCountEstimate();
double actualError = (double) (testSet.size() - estimate) / testSet.size();
- System.out.println(testSet.size() + "-" + estimate);
+ System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
- System.out.println("=" + actualError);
if (Math.abs(actualError) > errorRate) {
errorCount1++;
}
[17/17] incubator-kylin git commit: Merge branch '0.7-staging' into
v1.0-release
Posted by lu...@apache.org.
Merge branch '0.7-staging' into v1.0-release
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f03dfedb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f03dfedb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f03dfedb
Branch: refs/heads/v1.0-release
Commit: f03dfedbf447030b59dcad18ec5cf91576b1bfc7
Parents: 80793fd 19e4c23
Author: Luke Han <lu...@apache.org>
Authored: Thu Aug 20 15:57:21 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Thu Aug 20 15:57:21 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 15 +-
.../apache/kylin/common/util/HbaseImporter.java | 9 +-
.../common/util/HyperLogLogCounterTest.java | 10 +-
.../minicluster/hbase-export.tar.gz | Bin 22048217 -> 27762040 bytes
.../test_case_data/minicluster/kylin.properties | 2 +-
.../kylin/job/constant/BatchConstants.java | 33 ++--
.../kylin/job/hadoop/AbstractHadoopJob.java | 45 +++--
.../cardinality/HiveColumnCardinalityJob.java | 3 +-
.../job/hadoop/cube/FactDistinctColumnsJob.java | 10 +-
.../kylin/job/hadoop/cube/NDCuboidMapper.java | 4 +-
.../job/hadoop/cube/NewBaseCuboidMapper.java | 2 +-
.../kylin/job/hadoop/hbase/CreateHTableJob.java | 4 -
.../apache/kylin/job/BuildIIWithEngineTest.java | 1 +
.../java/org/apache/kylin/job/DeployUtil.java | 2 +-
.../org/apache/kylin/job/ExportHBaseData.java | 41 +++--
.../org/apache/kylin/job/ImportHBaseData.java | 174 -------------------
.../job/hadoop/cube/MergeCuboidJobTest.java | 8 +-
.../apache/kylin/query/test/IIQueryTest.java | 2 -
script/compress.sh | 2 +-
.../apache/kylin/rest/service/CacheService.java | 7 +-
.../apache/kylin/rest/service/CubeService.java | 3 -
website/_docs/gettingstarted/events.md | 1 +
22 files changed, 97 insertions(+), 281 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f03dfedb/script/compress.sh
----------------------------------------------------------------------
[03/17] incubator-kylin git commit: Fix KYLIN-949 by adding
hashCode() and equals() method in PrepareSqlRequest
Posted by lu...@apache.org.
Fix KYLIN-949 by adding hashCode() and equals() method in PrepareSqlRequest
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/752e307a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/752e307a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/752e307a
Branch: refs/heads/v1.0-release
Commit: 752e307a60a79d464a03ad06f700c7d701c67e61
Parents: 0375fa7
Author: Huang, Hua <hu...@mininglamp.com>
Authored: Fri Aug 14 17:20:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Aug 14 17:53:39 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/request/PrepareSqlRequest.java | 57 ++++++++++++++++++++
1 file changed, 57 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/752e307a/server/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java b/server/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
index 6d694f7..19b952e 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.request;
+import java.util.Arrays;
+
/**
* @author xduo
*
@@ -60,5 +62,60 @@ public class PrepareSqlRequest extends SQLRequest {
public void setValue(String value) {
this.value = value;
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((className == null) ? 0 : className.hashCode());
+ result = prime * result + ((value == null) ? 0 : value.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ StateParam other = (StateParam)obj;
+ if (className == null) {
+ if (other.className != null)
+ return false;
+ } else if (!className.equals(other.className))
+ return false;
+ if (value == null) {
+ if (other.value != null)
+ return false;
+ } else if (!value.equals(other.value))
+ return false;
+ return true;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((params == null) ? 0 : Arrays.hashCode(params));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ if (!super.equals(obj))
+ return false;
+ PrepareSqlRequest other = (PrepareSqlRequest)obj;
+ if (!Arrays.equals(params, other.params))
+ return false;
+ return true;
}
}
[14/17] incubator-kylin git commit: hot fix: mvn test with minicluster
Posted by lu...@apache.org.
hot fix: mvn test with minicluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ebfa38d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ebfa38d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ebfa38d1
Branch: refs/heads/v1.0-release
Commit: ebfa38d14d5a0ca638e1663ee2b569fc96753d8f
Parents: 36f3665
Author: honma <ho...@ebay.com>
Authored: Wed Aug 19 20:11:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 19 20:11:46 2015 +0800
----------------------------------------------------------------------
.../minicluster/hbase-export.tar.gz | Bin 22048217 -> 27762040 bytes
.../test_case_data/minicluster/kylin.properties | 2 +-
2 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ebfa38d1/examples/test_case_data/minicluster/hbase-export.tar.gz
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/hbase-export.tar.gz b/examples/test_case_data/minicluster/hbase-export.tar.gz
index 9a327a2..5800a56 100755
Binary files a/examples/test_case_data/minicluster/hbase-export.tar.gz and b/examples/test_case_data/minicluster/hbase-export.tar.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ebfa38d1/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index e81b8d5..62d1dab 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -27,7 +27,7 @@ kylin.metadata.url=kylin_metadata@hbase:minicluster:/hbase-unsecure
kylin.storage.url=hbase:minicluster:/hbase-unsecure
# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
+kylin.hdfs.working.dir=/tmp
kylin.job.mapreduce.default.reduce.input.mb=500
[04/17] incubator-kylin git commit: new blog, Dictionary in Kylin
Posted by lu...@apache.org.
new blog, Dictionary in Kylin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2f23dfed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2f23dfed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2f23dfed
Branch: refs/heads/v1.0-release
Commit: 2f23dfedb4c4bc3d17f8c027e17febe1308f920e
Parents: 752e307
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Aug 14 18:04:33 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Aug 14 18:06:08 2015 +0800
----------------------------------------------------------------------
.../_posts/blog/2015-08-13-kylin-dictionary.md | 68 +++++++++++++++++++
.../images/develop/dictionary-bytes-format.png | Bin 0 -> 29293 bytes
.../images/develop/dictionary-cache-layer.png | Bin 0 -> 10883 bytes
.../images/develop/dictionary-trie-sample.png | Bin 0 -> 19555 bytes
4 files changed, 68 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2f23dfed/website/_posts/blog/2015-08-13-kylin-dictionary.md
----------------------------------------------------------------------
diff --git a/website/_posts/blog/2015-08-13-kylin-dictionary.md b/website/_posts/blog/2015-08-13-kylin-dictionary.md
new file mode 100644
index 0000000..7e3c8b8
--- /dev/null
+++ b/website/_posts/blog/2015-08-13-kylin-dictionary.md
@@ -0,0 +1,68 @@
+---
+layout: post-blog
+title: "Dictionary in Kylin"
+date: 2015-08-13 14:37:00
+author: Li Yang
+categories: blog
+---
+
+### Purpose of Dictionary
+
+Dictionary is a [classic compression technique](https://en.wikipedia.org/wiki/Dictionary_coder) that can greatly reduce the size of data. Kylin apply dictionary to all dimension values stored in cube.
+
+Kylin's requirement to dictionary:
+
+* Compress cube size by storing IDs instead of real values
+* Bi-way mapping of dimension values from/to IDs
+* Preserving order to facilitate range query
+* Minimal memory & storage footprint
+
+### Dictionary Design
+
+Dictionary is implemented as a [trie](https://en.wikipedia.org/wiki/Trie) data structure. Dictionary ID (or "seq. no" below) is chosen in a way to preserve value order. Then at query time, predicate filters can be pushed down to storage and be evaluated on the IDs.
+
+* Trie node are labeled by 1) number of values underneath; 2) is end of value or not
+* Bi-way lookup between "value" <==> "seq. no" by top-down navigate
+* The "seq. no" preserves value order and is a minimal integer for space advantage
+* O(L) lookup time, where L=max(value length)
+
+An example of a trie dictionary.
+
+
+
+### Memory structure
+
+Once built, the dictionary is serialized into a chunk of bytes. This is how it stays in memory and also in file.
+
+* Head
+ * magic, head len, body len, child_offset size, seq_no size, base ID, max value len, bytes converter
+* Body
+ * a flattened trie, where each node is
+ * child offset (size specified in head)
+ * 1st MSB: isLastChild
+ * 2nd MSB: isEndOfValue
+ * no. values beneath (size specified in head)
+ * value len (1 byte unsigned)
+ * value bytes
+
+
+
+### Benchmark result
+
+We compared dictionary's size and performance with HashMap and ID Based Array. It's memory footprint is an order less and the throughput is very stable accross scales.
+
+| | HashMap (value=>id) | Dictionary (value=>id) | IdArray (id=>value) | Dictionary (id=>value) | |
+| ------------------------------------|---------------------|------------------------|---------------------|------------------------|----------------------|
+| 150K eng words footprint (bytes) | 18.8M | *1.7M* | 11.1M | *1.7M* | 1.4M raw size |
+| 150K eng words throughput (acc/s) | 13M | 1.9M | 150M | 1.96M | 31 max value len |
+| 6.6K categories footprint (bytes) | 0.94M | 0.13M | 0.58M | 0.12M | 0.1M raw size |
+| 6.6K categories throughput (acc/s) | 26M | 2.0M | 98M | 2.0M | 30 max value len |
+| 6 words footprint (bytes) | 792B | 168B | 416B | 168B | 33B raw size |
+| 6 works throughput (acc/s) | 68.5M | 14.7M | 714M | 11.1M | 9 max value len |
+
+### Cache layer
+
+To achieve maximum lookup throughput, a cache layer (HashMap or IdArray) sits on top of dictionary using weak reference. The cache could be gone when memory runs short, then dictionary will be hit directly.
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2f23dfed/website/images/develop/dictionary-bytes-format.png
----------------------------------------------------------------------
diff --git a/website/images/develop/dictionary-bytes-format.png b/website/images/develop/dictionary-bytes-format.png
new file mode 100644
index 0000000..d3641f6
Binary files /dev/null and b/website/images/develop/dictionary-bytes-format.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2f23dfed/website/images/develop/dictionary-cache-layer.png
----------------------------------------------------------------------
diff --git a/website/images/develop/dictionary-cache-layer.png b/website/images/develop/dictionary-cache-layer.png
new file mode 100644
index 0000000..99b3dcc
Binary files /dev/null and b/website/images/develop/dictionary-cache-layer.png differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2f23dfed/website/images/develop/dictionary-trie-sample.png
----------------------------------------------------------------------
diff --git a/website/images/develop/dictionary-trie-sample.png b/website/images/develop/dictionary-trie-sample.png
new file mode 100644
index 0000000..7b63d04
Binary files /dev/null and b/website/images/develop/dictionary-trie-sample.png differ
[06/17] incubator-kylin git commit: KYLIN-946,
fix error when refresh [Tables] page
Posted by lu...@apache.org.
KYLIN-946, fix error when refresh [Tables] page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/39bd19a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/39bd19a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/39bd19a9
Branch: refs/heads/v1.0-release
Commit: 39bd19a9dd0d3c1fcb86a312775c8d70211339d7
Parents: b549275
Author: jiazhong <ji...@ebay.com>
Authored: Fri Aug 14 18:46:31 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Fri Aug 14 18:47:54 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/sourceMeta.js | 12 +++++++++---
webapp/app/js/model/projectModel.js | 3 +++
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39bd19a9/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index 85307f8..d05dc7f 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -43,6 +43,11 @@ KylinApp
};
$scope.aceSrcTbLoaded = function (forceLoad) {
+ //stop request when project invalid
+ if(!$scope.projectModel.getSelectedProject()){
+ TableModel.init();
+ return;
+ }
if (forceLoad) {
$httpDefaultCache.removeAll();
}
@@ -53,11 +58,12 @@ KylinApp
SweetAlert.swal('Oops...',resp, 'error');
});
};
-
+ $scope.aceSrcTbLoaded();
$scope.$watch('projectModel.selectedProject', function (newValue, oldValue) {
// will load table when enter this page,null or not
- $scope.aceSrcTbLoaded();
-
+ if (newValue != oldValue || newValue == null) {
+ $scope.aceSrcTbLoaded();
+ }
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/39bd19a9/webapp/app/js/model/projectModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/projectModel.js b/webapp/app/js/model/projectModel.js
index a23fba5..8f466e7 100644
--- a/webapp/app/js/model/projectModel.js
+++ b/webapp/app/js/model/projectModel.js
@@ -28,6 +28,9 @@ KylinApp.service('ProjectModel', function () {
}
};
this.getSelectedProject = function (project) {
+ if(this.selectedProject=="_null"){
+ return null;
+ }
return this.selectedProject;
};
[13/17] incubator-kylin git commit: add slide at hadoop summit
shanghai 2015
Posted by lu...@apache.org.
add slide at hadoop summit shanghai 2015
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/36f3665c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/36f3665c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/36f3665c
Branch: refs/heads/v1.0-release
Commit: 36f3665c2c4a72f669c2ec7dfc0006a107a0fa01
Parents: 0150b51
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Aug 19 17:35:44 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Aug 19 17:36:02 2015 +0800
----------------------------------------------------------------------
website/_docs/gettingstarted/events.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/36f3665c/website/_docs/gettingstarted/events.md
----------------------------------------------------------------------
diff --git a/website/_docs/gettingstarted/events.md b/website/_docs/gettingstarted/events.md
index 8190184..8605d4a 100644
--- a/website/_docs/gettingstarted/events.md
+++ b/website/_docs/gettingstarted/events.md
@@ -11,6 +11,7 @@ __Coming Events__
__Conferences__
+* [Apache Kylin - Balance Between Space and Time](http://www.chinahadoop.com/2015/July/Shanghai/agenda.php) ([slides](http://www.slideshare.net/qhzhou/apache-kylin-china-hadoop-summit-2015-shanghai)) by [Qianhao Zhou](https://github.com/qhzhou), at Hadoop Summit 2015 in Shanghai, China, 2015-07-24
* [Apache Kylin - Balance Between Space and Time](https://www.youtube.com/watch?v=jgvZSFaXPgI), [slides](http://www.slideshare.net/DebashisSaha/apache-kylin-balance-between-space-and-time-hadop-summit-2015) ([video](https://www.youtube.com/watch?v=jgvZSFaXPgI), [slides](http://www.slideshare.net/DebashisSaha/apache-kylin-balance-between-space-and-time-hadop-summit-2015)) by [Debashis Saha](https://twitter.com/debashis_saha) & [Luke Han](https://twitter.com/lukehq), at Hadoop Summit 2015 in San Jose, US, 2015-06-09
* [HBaseCon 2015: Apache Kylin; Extreme OLAP Engine for Hadoop](https://vimeo.com/128152444) ([video](https://vimeo.com/128152444), [slides](http://www.slideshare.net/HBaseCon/ecosystem-session-3b)) by [Seshu Adunuthula](https://twitter.com/SeshuAd) at HBaseCon 2015 in San Francisco, US, 2015-05-07
* [Apache Kylin - Extreme OLAP Engine for Hadoop](http://strataconf.com/big-data-conference-uk-2015/public/schedule/detail/40029) ([slides](http://www.slideshare.net/lukehan/apache-kylin-extreme-olap-engine-for-big-data)) by [Luke Han](https://twitter.com/lukehq) & [Yang Li](https://github.com/liyang-gmt8), at Strata+Hadoop World in London, UK, 2015-05-06
[08/17] incubator-kylin git commit: update doc,
consume rest api in js , and tables page update
Posted by lu...@apache.org.
update doc,consume rest api in js ,and tables page update
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4ab6372c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4ab6372c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4ab6372c
Branch: refs/heads/v1.0-release
Commit: 4ab6372c88282831308443938155e7758f8bbba0
Parents: 265a528
Author: jiazhong <ji...@ebay.com>
Authored: Mon Aug 17 15:21:55 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Mon Aug 17 15:22:33 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/sourceMeta.js | 6 ++++++
webapp/app/partials/tables/source_table_tree.html | 2 +-
website/_docs/howto/howto_use_restapi_in_js.md | 2 +-
3 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4ab6372c/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index d05dc7f..554e128 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -76,6 +76,12 @@ KylinApp
}
};
+ $scope.selectedNode = function(obj){
+ if (obj.uuid) {
+ return $scope.tableModel.selectedSrcTable;
+ }
+ }
+
$scope.aceSrcTbChanged = function () {
$scope.tableModel.selectedSrcDb = [];
$scope.tableModel.selectedSrcTable = {};
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4ab6372c/webapp/app/partials/tables/source_table_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/source_table_tree.html b/webapp/app/partials/tables/source_table_tree.html
index 2b40ab1..1e6e08f 100755
--- a/webapp/app/partials/tables/source_table_tree.html
+++ b/webapp/app/partials/tables/source_table_tree.html
@@ -39,7 +39,7 @@
tree-model="tableModel.selectedSrcDb"
options="tableModel.treeOptions"
on-selection="showSelected(node)"
- selected-node="tableModel.selectedSrcTable">
+ selected-node="selectedNode(node)">
{{node.name}} {{!!(node.datatype)?'(' + trimType(node.datatype) + ')' : ''}}
</treecontrol>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4ab6372c/website/_docs/howto/howto_use_restapi_in_js.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_use_restapi_in_js.md b/website/_docs/howto/howto_use_restapi_in_js.md
index 54b2c3e..45c8974 100644
--- a/website/_docs/howto/howto_use_restapi_in_js.md
+++ b/website/_docs/howto/howto_use_restapi_in_js.md
@@ -16,7 +16,7 @@ $.ajaxSetup({
var request = $.ajax({
url: "http://hostname/kylin/api/query",
type: "POST",
- data: '{"sql":"select count(*) from SUMMARY;","offset":0,"limit":50000,"partial":true,"project":"test"}',
+ data: '{"sql":"select count(*) from SUMMARY;","offset":0,"limit":50000,"acceptPartial":true,"project":"test"}',
dataType: "json"
});
request.done(function( msg ) {
[12/17] incubator-kylin git commit: hot fix: compress in commit sha
Posted by lu...@apache.org.
hot fix: compress in commit sha
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0150b515
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0150b515
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0150b515
Branch: refs/heads/v1.0-release
Commit: 0150b51585bdc61b2410fa90866c91a28b4a2510
Parents: cc4fdf1
Author: honma <ho...@ebay.com>
Authored: Wed Aug 19 16:36:07 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 19 16:36:24 2015 +0800
----------------------------------------------------------------------
script/compress.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0150b515/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index b1c51cd..5b773a1 100755
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -32,7 +32,7 @@ echo 'package tar.gz'
rm -rf dist/kylin-${version}
mkdir dist/kylin-${version}
# copy files
-cp -r lib bin conf dist/tomcat examples/sample_cube commit.sha1 dist/kylin-${version}
+cp -r lib bin conf dist/tomcat examples/sample_cube dist/commit.sha1 dist/kylin-${version}
# copy license files
cp LICENSE README.md NOTICE DISCLAIMER dist/kylin-${version}
[02/17] incubator-kylin git commit: Update metastore.sh to sync with
ResourceTool
Posted by lu...@apache.org.
Update metastore.sh to sync with ResourceTool
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0375fa7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0375fa7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0375fa7f
Branch: refs/heads/v1.0-release
Commit: 0375fa7f8bfcfbfa819bcbf0007838cebaad606c
Parents: b449590
Author: shaofengshi <sh...@apache.org>
Authored: Fri Aug 14 16:19:09 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Aug 14 16:19:56 2015 +0800
----------------------------------------------------------------------
bin/metastore.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0375fa7f/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
index 950a978..53aaa80 100755
--- a/bin/metastore.sh
+++ b/bin/metastore.sh
@@ -41,7 +41,7 @@ then
echo "Starting backup to ${_file}"
mkdir -p ${_file}
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy ${KYLIN_HOME}/conf/kylin.properties ${_file}
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool download ${_file}
echo "metadata store backed up to ${_file}"
elif [ $1 == "restore" ]
@@ -49,7 +49,7 @@ then
_file=$2
echo "Starting restoring $_file"
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool copy $_file ${KYLIN_HOME}/conf/kylin.properties
+ hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool upload $_file
elif [ $1 == "reset" ]
then
[05/17] incubator-kylin git commit: minor,
add comment in kylin.sh for troubleshooting
Posted by lu...@apache.org.
minor, add comment in kylin.sh for troubleshooting
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b549275c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b549275c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b549275c
Branch: refs/heads/v1.0-release
Commit: b549275c55981b787a5e1e0a8d940b5af2ba38af
Parents: 2f23dfe
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Aug 14 18:05:37 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Aug 14 18:06:13 2015 +0800
----------------------------------------------------------------------
bin/kylin.sh | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b549275c/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 94f9143..dcd835e 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -52,6 +52,9 @@ then
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+ #debug if encounter NoClassDefError
+ #hbase classpath
hbase ${KYLIN_EXTRA_START_OPTS} \
-Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
[16/17] incubator-kylin git commit: KYLIN-954 prepare for v1.0 release
Posted by lu...@apache.org.
KYLIN-954 prepare for v1.0 release
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/80793fd0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/80793fd0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/80793fd0
Branch: refs/heads/v1.0-release
Commit: 80793fd0dba72aa6d798698f675646ae16502b88
Parents: 2beba36
Author: Luke Han <lu...@apache.org>
Authored: Thu Aug 20 15:56:21 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Thu Aug 20 15:56:21 2015 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 2 +-
common/pom.xml | 2 +-
cube/pom.xml | 2 +-
dictionary/pom.xml | 2 +-
invertedindex/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
job/pom.xml | 2 +-
metadata/pom.xml | 2 +-
monitor/pom.xml | 2 +-
pom.xml | 2 +-
query/pom.xml | 2 +-
script/compress.sh | 4 ++--
server/pom.xml | 2 +-
storage/pom.xml | 2 +-
website/download/index.md | 6 +++---
15 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index d4209fb..6e8d025 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index ff69208..acd3e0a 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/cube/pom.xml
----------------------------------------------------------------------
diff --git a/cube/pom.xml b/cube/pom.xml
index 450d8fa..5fbdc7c 100644
--- a/cube/pom.xml
+++ b/cube/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/dictionary/pom.xml b/dictionary/pom.xml
index 5078d56..85cdde6 100644
--- a/dictionary/pom.xml
+++ b/dictionary/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/invertedindex/pom.xml
----------------------------------------------------------------------
diff --git a/invertedindex/pom.xml b/invertedindex/pom.xml
index 0a00d24..42b4aa6 100644
--- a/invertedindex/pom.xml
+++ b/invertedindex/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-invertedindex</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index ccdac54..33378d4 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
index 0aed7ac..cbbcfaa 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>kylin-job</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/metadata/pom.xml
----------------------------------------------------------------------
diff --git a/metadata/pom.xml b/metadata/pom.xml
index d2e6a18..55344b9 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -27,7 +27,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/monitor/pom.xml
----------------------------------------------------------------------
diff --git a/monitor/pom.xml b/monitor/pom.xml
index 421e063..e9beca9 100644
--- a/monitor/pom.xml
+++ b/monitor/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8e118b0..d45b3d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
<name>Kylin:HadoopOLAPEngine</name>
<url>http://kylin.incubator.apache.org</url>
<inceptionYear>2014</inceptionYear>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index b9d20d8..965e8c6 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index b1c51cd..27af365 100755
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -40,7 +40,7 @@ find dist/kylin-${version} -type d -exec chmod 755 {} \;
find dist/kylin-${version} -type f -exec chmod 644 {} \;
find dist/kylin-${version} -type f -name "*.sh" -exec chmod 755 {} \;
cd dist
-tar -cvzf kylin-${version}.tar.gz kylin-${version}
+tar -cvzf kylin-${version}-bin.tar.gz kylin-${version}
rm -rf kylin-${version}
-echo "Package ready dist/kylin-${version}.tar.gz"
+echo "Package ready dist/kylin-${version}-bin.tar.gz"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index b4b1043..d44322c 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/storage/pom.xml
----------------------------------------------------------------------
diff --git a/storage/pom.xml b/storage/pom.xml
index 601af3f..30905b9 100644
--- a/storage/pom.xml
+++ b/storage/pom.xml
@@ -26,7 +26,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>0.7.3-incubating-SNAPSHOT</version>
+ <version>1.0-incubating-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/80793fd0/website/download/index.md
----------------------------------------------------------------------
diff --git a/website/download/index.md b/website/download/index.md
index 294cc9c..9ecc01d 100644
--- a/website/download/index.md
+++ b/website/download/index.md
@@ -32,9 +32,9 @@ And ODBC Driver could be find here:
__Development Version (v0.7.3)__
Here's latest development version binary package:
- * [kylin-v0.7.3-incubating-SNAPSHOT](kylin-0.7.3-incubating-SNAPSHOT.tar.gz)
- * Updated Date: 2015-08-04
- * Commit ID: a880dd4fb6e4443d2bcad0d57db3ae09bac65c4d
+ * [kylin-v1.0-incubating-SNAPSHOT](kylin-1.0-incubating-SNAPSHOT-bin.tar.gz)
+ * Updated Date: 2015-08-14
+ * Commit ID: b449590cc5c6aaa3eea9d19f922473bff6fce4b0