You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/12 06:42:51 UTC
[01/50] incubator-kylin git commit: fix bug
Repository: incubator-kylin
Updated Branches:
refs/heads/inverted-index 4056fc379 -> 355a09b58
fix bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8b1e74f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8b1e74f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8b1e74f6
Branch: refs/heads/inverted-index
Commit: 8b1e74f6ee17299a05daa13e05b4399ed53a1f5d
Parents: 80e9cc9
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 12:30:05 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 12:30:05 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/common/MapReduceExecutable.java | 12 +++++++++++-
.../org/apache/kylin/job/tools/HadoopStatusChecker.java | 5 +++--
sbin/start-kylin-sandbox.sh | 2 +-
sbin/start-kylin.sh | 2 +-
4 files changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8b1e74f6/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
index 09b3011..20791a0 100644
--- a/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
+++ b/job/src/main/java/org/apache/kylin/job/common/MapReduceExecutable.java
@@ -19,6 +19,7 @@
package org.apache.kylin.job.common;
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.constant.JobStepStatusEnum;
import org.apache.kylin.job.exception.ExecuteException;
@@ -116,7 +117,16 @@ public class MapReduceExecutable extends AbstractExecutable {
}
final StringBuilder output = new StringBuilder();
final HadoopCmdOutput hadoopCmdOutput = new HadoopCmdOutput(job, output);
- final String rmWebHost = job.getConfiguration().get("yarn.resourcemanager.webapp.address");
+ String rmWebHost = job.getConfiguration().get("yarn.resourcemanager.webapp.address");
+ if (StringUtils.isEmpty(rmWebHost)) {
+ return new ExecuteResult(ExecuteResult.State.ERROR, "yarn.resourcemanager.webapp.address is empty");
+ }
+ if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
+ //do nothing
+ } else {
+ rmWebHost = "http://" + rmWebHost;
+ }
+ logger.info("yarn.resourcemanager.webapp.address:" + rmWebHost);
final String restStatusCheckUrl = rmWebHost + "/ws/v1/cluster/apps/${job_id}?anonymous=true";
String mrJobId = hadoopCmdOutput.getMrJobId();
HadoopStatusChecker statusChecker = new HadoopStatusChecker(restStatusCheckUrl, mrJobId, output);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8b1e74f6/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
index 25d99a5..31cff50 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusChecker.java
@@ -34,7 +34,7 @@ import java.util.Date;
*/
public class HadoopStatusChecker {
- protected static final Logger log = LoggerFactory.getLogger(HadoopStatusChecker.class);
+ protected static final Logger logger = LoggerFactory.getLogger(HadoopStatusChecker.class);
private final String yarnUrl;
private final String mrJobID;
@@ -54,7 +54,7 @@ public class HadoopStatusChecker {
JobStepStatusEnum status = null;
try {
final Pair<RMAppState, FinalApplicationStatus> result = new HadoopStatusGetter(yarnUrl, mrJobID).get();
- log.debug("State of Hadoop job: " + mrJobID + ":" + result.getLeft() + "-" + result.getRight());
+ logger.debug("State of Hadoop job: " + mrJobID + ":" + result.getLeft() + "-" + result.getRight());
output.append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").format(new Date()) + " - State of Hadoop job: " + mrJobID + ":" + result.getLeft() + " - " + result.getRight() + "\n");
switch (result.getRight()) {
@@ -88,6 +88,7 @@ public class HadoopStatusChecker {
break;
}
} catch (Exception e) {
+ logger.error("error check status", e);
output.append("Exception: " + e.getLocalizedMessage() + "\n");
status = JobStepStatusEnum.ERROR;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8b1e74f6/sbin/start-kylin-sandbox.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin-sandbox.sh b/sbin/start-kylin-sandbox.sh
index 5e436fe..b8912f2 100644
--- a/sbin/start-kylin-sandbox.sh
+++ b/sbin/start-kylin-sandbox.sh
@@ -32,7 +32,7 @@ cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
source ${dir}/find-hive-dependency.sh
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
-export HBASE_CLASSPATH=$hive_dependency
+export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8b1e74f6/sbin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin.sh b/sbin/start-kylin.sh
index 727fe33..d7436f4 100644
--- a/sbin/start-kylin.sh
+++ b/sbin/start-kylin.sh
@@ -32,7 +32,7 @@ cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
source ${dir}/find-hive-dependency.sh
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
-export HBASE_CLASSPATH=$hive_dependency
+export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
[46/50] incubator-kylin git commit: KYLIN-608 bug fix
Posted by li...@apache.org.
KYLIN-608 bug fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a864ba2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a864ba2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a864ba2d
Branch: refs/heads/inverted-index
Commit: a864ba2d2c3e7354dac0f66e7c315563a0e71b88
Parents: ecfc3cc
Author: honma <ho...@ebay.com>
Authored: Thu Feb 12 13:02:57 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Feb 12 13:02:57 2015 +0800
----------------------------------------------------------------------
.../coprocessor/endpoint/EndpointAggregators.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a864ba2d/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
index c6d8c49..516c160 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
@@ -157,6 +157,14 @@ public class EndpointAggregators {
rawTableRecord.setBytes(row, 0, row.length);
for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
+ if (metricInfos[metricIndex].type == MetricType.Count) {
+ measureAggrs[metricIndex].aggregate(ONE);
+ continue;
+ }
+
+ if (metricInfos[metricIndex].type == MetricType.DimensionAsMetric) {
+ continue;
+ }
MetricInfo metricInfo = metricInfos[metricIndex];
MeasureAggregator aggregator = measureAggrs[metricIndex];
@@ -178,13 +186,6 @@ public class EndpointAggregators {
aggregator.aggregate(hllc);
}
}
-
- //aggregate for "count"
- for (int i = 0; i < metricInfos.length; ++i) {
- if (metricInfos[i].type == MetricType.Count) {
- measureAggrs[i].aggregate(ONE);
- }
- }
}
/**
[48/50] incubator-kylin git commit: small patch to package scripts
Posted by li...@apache.org.
small patch to package scripts
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/be5e23f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/be5e23f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/be5e23f2
Branch: refs/heads/inverted-index
Commit: be5e23f2200f907c67dd4997bbf416a44072e2a8
Parents: 4726314
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Feb 12 13:20:11 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Feb 12 13:20:11 2015 +0800
----------------------------------------------------------------------
script/build.sh | 12 ++++++------
script/compress.sh | 4 +++-
script/prepare.sh | 6 +++++-
3 files changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be5e23f2/script/build.sh
----------------------------------------------------------------------
diff --git a/script/build.sh b/script/build.sh
index 3014123..de6f76c 100644
--- a/script/build.sh
+++ b/script/build.sh
@@ -5,13 +5,13 @@ echo "package back-end"
dir=$(dirname ${0})
cd ${dir}/..
-mvn clean install -DskipTests
+mvn clean install -DskipTests || { exit 1; }
#package webapp
echo 'package front-end'
cd webapp
-npm install -g bower
-bower --allow-root install
-npm install
-npm install -g grunt-cli
-grunt dev --buildEnv=dev
+npm install -g bower || { exit 1; }
+bower --allow-root install || { exit 1; }
+npm install || { exit 1; }
+npm install -g grunt-cli || { exit 1; }
+grunt dev --buildEnv=dev || { exit 1; }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be5e23f2/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index 20b4257..4dd91f3 100644
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -17,4 +17,6 @@ rm -rf kylin-${version}
mkdir kylin-${version}
cp -r lib bin conf tomcat kylin-${version}
tar -cvzf kylin-${version}.tgz kylin-${version}
-rm -rf kylin-${version}
\ No newline at end of file
+rm -rf kylin-${version}
+
+echo "Package ready kylin-${version}.tgz"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/be5e23f2/script/prepare.sh
----------------------------------------------------------------------
diff --git a/script/prepare.sh b/script/prepare.sh
index 3b30ac2..a1c8537 100644
--- a/script/prepare.sh
+++ b/script/prepare.sh
@@ -7,8 +7,8 @@ if [ -z "$version" ]
then
echo 'version not set'
version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
- echo "${version}"
fi
+echo "version ${version}"
echo "copy lib file"
rm -rf lib
@@ -16,6 +16,10 @@ mkdir lib
cp server/target/kylin-server-${version}.war tomcat/webapps/kylin.war
cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
+# Copied file becomes 000 for some env (e.g. my Cygwin)
+chmod 644 tomcat/webapps/kylin.war
+chmod 644 lib/kylin-job-${version}.jar
+chmod 644 lib/kylin-coprocessor-${version}.jar
echo "add js css to war"
if [ ! -d "webapp/dist" ]
[37/50] incubator-kylin git commit: Update dictionary file
Posted by li...@apache.org.
Update dictionary file
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8245337e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8245337e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8245337e
Branch: refs/heads/inverted-index
Commit: 8245337eee917fdb66b11bbec63975bde85f7b31
Parents: 7fefad5
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Feb 11 16:09:20 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Feb 11 16:09:20 2015 +0800
----------------------------------------------------------------------
.../16d8185c-ee6b-4f8c-a919-756d9809f937.dict | Bin 2515 -> 2630 bytes
.../c12ae49d-9dbe-4a58-b169-19afac317696.dict | Bin 2515 -> 2630 bytes
.../eaed91b0-4182-4ee5-a733-1047a622ee29.dict | Bin 2515 -> 2630 bytes
.../ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict | Bin 2515 -> 2630 bytes
.../7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict | Bin 2940 -> 3105 bytes
.../8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict | Bin 2940 -> 3105 bytes
.../ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict | Bin 2940 -> 3105 bytes
.../eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict | Bin 2940 -> 3105 bytes
.../2602386c-debb-4968-8d2f-b52b8215e385.dict | Bin 1841 -> 2000 bytes
.../4243889f-bc81-4807-a975-7041bbbf35e7.dict | Bin 1841 -> 2000 bytes
.../494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict | Bin 1841 -> 2000 bytes
.../8b4b1c06-fb74-486b-a2ad-74420afebcda.dict | Bin 1841 -> 2000 bytes
.../ac520edd-f9d2-419f-a7de-587bfb97dc81.dict | Bin 1841 -> 2000 bytes
.../0410d2c4-4686-40bc-ba14-170042a2de94.dict | Bin 1498 -> 1537 bytes
.../51ff5e6c-22b3-444e-9915-d376a10f20cb.dict | Bin 1498 -> 1537 bytes
.../a4e57e55-48fc-4f25-a9c8-485deed25925.dict | Bin 1498 -> 1537 bytes
.../aceae914-4246-4251-a0c2-692fe7a300df.dict | Bin 1498 -> 1537 bytes
.../b298089f-9656-4693-b9b2-8fea46f06dd5.dict | Bin 1498 -> 1537 bytes
.../c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict | Bin 1498 -> 1537 bytes
.../0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict | Bin 661 -> 640 bytes
.../14fe66b3-5956-498c-bd93-40182cac5510.dict | Bin 661 -> 640 bytes
.../1d383a36-81b9-4177-a822-04eab3683e5b.dict | Bin 661 -> 640 bytes
.../c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict | Bin 661 -> 640 bytes
.../0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict | Bin 2189 -> 2264 bytes
.../31edf35b-ffca-4f24-8229-f87dc34e3087.dict | Bin 2189 -> 2264 bytes
.../652bd393-678a-4f16-a504-fd8ce1229355.dict | Bin 2189 -> 2264 bytes
.../792eb972-d046-48e6-9428-f6a3aed92fad.dict | Bin 2189 -> 2264 bytes
.../914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict | Bin 2189 -> 2264 bytes
.../d25e554e-deac-4e4a-9289-96f0d121d8fc.dict | Bin 2189 -> 2264 bytes
29 files changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict
index a2cb760..6d5d786 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict
index e56eefb..8613105 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/c12ae49d-9dbe-4a58-b169-19afac317696.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict
index efcacde..bf55928 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/eaed91b0-4182-4ee5-a733-1047a622ee29.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict
index 95cf51c..ab6471e 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict
index 823c00d..6cc0bd3 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict
index 5236f78..45196fa 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict
index bc1d8e8..6f76b45 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict
index 0bbe66f..5c3713d 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict
index 1d25663..ccb833b 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/2602386c-debb-4968-8d2f-b52b8215e385.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict
index a046dbb..cf7a4b4 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict
index c1a4196..106eb76 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict
index b60e34f..6027c81 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/8b4b1c06-fb74-486b-a2ad-74420afebcda.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict
index 2c47f75..b7946cf 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/ac520edd-f9d2-419f-a7de-587bfb97dc81.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict
index c72d118..63f659b 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/0410d2c4-4686-40bc-ba14-170042a2de94.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict
index 4fd22f8..4c8dfbe 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/51ff5e6c-22b3-444e-9915-d376a10f20cb.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict
index 67dbea4..1fd7418 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/a4e57e55-48fc-4f25-a9c8-485deed25925.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict
index df0c657..147986e 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict
index 3a28967..26dda49 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/b298089f-9656-4693-b9b2-8fea46f06dd5.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict
index e898c45..36523c0 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict and b/examples/test_case_data/localmeta/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict
index afe548d..0e03620 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict and b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict
index 460da2b..8e92724 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict and b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/14fe66b3-5956-498c-bd93-40182cac5510.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict
index 7a97f71..400ebf1 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict and b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/1d383a36-81b9-4177-a822-04eab3683e5b.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict
index 6f270fe..91de7c8 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict and b/examples/test_case_data/localmeta/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict
index 5be36a1..00be694 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict
index 87057cd..c9a6a66 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/31edf35b-ffca-4f24-8229-f87dc34e3087.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict
index 194b7e2..cea96a2 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict
index 0a60fc3..e3b4784 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/792eb972-d046-48e6-9428-f6a3aed92fad.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict
index eb64935..1b0c03b 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8245337e/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict
index a10c74b..700f6b5 100644
Binary files a/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict and b/examples/test_case_data/localmeta/dict/TEST_SITES/SITE_ID/d25e554e-deac-4e4a-9289-96f0d121d8fc.dict differ
[22/50] incubator-kylin git commit: merge conflict
Posted by li...@apache.org.
merge conflict
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/59411d92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/59411d92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/59411d92
Branch: refs/heads/inverted-index
Commit: 59411d923ffad52d75efc22d15b6176c520a678e
Parents: a2383a5
Author: jiazhong <ji...@ebay.com>
Authored: Wed Feb 11 10:28:15 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Wed Feb 11 10:28:15 2015 +0800
----------------------------------------------------------------------
.gitignore | 0
bin/check-env.sh | 0
bin/start-kylin.sh | 0
bin/stop-kylin.sh | 0
conf/kylin.properties | 0
examples/test_case_data/sandbox-hdp21/kylin.properties | 0
examples/test_case_data/sandbox/kylin.properties | 0
script/package.sh | 0
webapp/app/css/AdminLTE-fonts.css | 0
webapp/app/css/AdminLTE.css | 0
.../ODelI1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2 | Bin
...BGKUt5Ot5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2 | Bin
webapp/app/index.html | 2 ++
webapp/app/js/controllers/cubeAdvanceSetting.js | 0
webapp/app/js/controllers/cubeDimensions.js | 0
webapp/app/js/controllers/cubeEdit.js | 0
webapp/app/js/controllers/cubeFilter.js | 0
webapp/app/js/controllers/cubeModel.js | 0
webapp/app/js/controllers/cubeRefresh.js | 0
webapp/app/js/controllers/cubeSchema.js | 0
webapp/app/js/controllers/page.js | 0
webapp/app/js/controllers/projects.js | 0
webapp/app/js/controllers/sourceMeta.js | 0
webapp/app/js/filters/filter.js | 0
webapp/app/js/model/cubeConfig.js | 0
webapp/app/js/model/cubesManager.js | 0
webapp/app/js/model/jobConfig.js | 0
webapp/app/js/model/metaModel.js | 0
webapp/app/js/model/projectConfig.js | 0
webapp/app/js/model/projectModel.js | 0
webapp/app/js/model/tableConfig.js | 0
webapp/app/js/model/tableModel.js | 0
webapp/app/partials/cubes/cube_json_edit.html | 0
webapp/app/partials/jobs/job_submit.html | 1 -
webapp/app/partials/projects/projects.html | 0
webapp/app/partials/tables/source_metadata.html | 0
webapp/bower.json | 0
webapp/grunt.json | 0
38 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/bin/check-env.sh b/bin/check-env.sh
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/css/AdminLTE-fonts.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE-fonts.css b/webapp/app/css/AdminLTE-fonts.css
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/css/AdminLTE.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/fonts/ODelI1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2
----------------------------------------------------------------------
diff --git a/webapp/app/fonts/ODelI1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2 b/webapp/app/fonts/ODelI1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/fonts/toadOcfmlt9b38dHJxOBGKUt5Ot5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2
----------------------------------------------------------------------
diff --git a/webapp/app/fonts/toadOcfmlt9b38dHJxOBGKUt5Ot5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2 b/webapp/app/fonts/toadOcfmlt9b38dHJxOBGKUt5Ot5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/webapp/app/index.html b/webapp/app/index.html
old mode 100755
new mode 100644
index f4c4b25..df87007
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -136,6 +136,8 @@
<script src="js/model/metaModel.js"></script>
<script src="js/model/projectModel.js"></script>
<script src="js/model/tableModel.js"></script>
+<script src="js/model/cubeListModel.js"></script>
+<script src="js/model/jobListModel.js"></script>
<script src="js/controllers/page.js"></script>
<script src="js/controllers/index.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeFilter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeFilter.js b/webapp/app/js/controllers/cubeFilter.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeRefresh.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeRefresh.js b/webapp/app/js/controllers/cubeRefresh.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/page.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/page.js b/webapp/app/js/controllers/page.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/projects.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/projects.js b/webapp/app/js/controllers/projects.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/cubesManager.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubesManager.js b/webapp/app/js/model/cubesManager.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/jobConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/jobConfig.js b/webapp/app/js/model/jobConfig.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/metaModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/metaModel.js b/webapp/app/js/model/metaModel.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/projectConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/projectConfig.js b/webapp/app/js/model/projectConfig.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/projectModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/projectModel.js b/webapp/app/js/model/projectModel.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/tableConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/tableConfig.js b/webapp/app/js/model/tableConfig.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/js/model/tableModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/tableModel.js b/webapp/app/js/model/tableModel.js
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/partials/cubes/cube_json_edit.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_json_edit.html b/webapp/app/partials/cubes/cube_json_edit.html
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/partials/jobs/job_submit.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/jobs/job_submit.html b/webapp/app/partials/jobs/job_submit.html
old mode 100755
new mode 100644
index 49af1e9..b3f917b
--- a/webapp/app/partials/jobs/job_submit.html
+++ b/webapp/app/partials/jobs/job_submit.html
@@ -35,7 +35,6 @@
<tr>
<td>Start Date (Include)</td>
<td>
- <!--cubes.js loadDetail function has convert from GMT mills to GMT string,reverse local -->
<!--no need convert UTC again-->
<div ng-if="cube.segments.length == 0">
{{jobBuildRequest.startTime =
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/partials/projects/projects.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/projects/projects.html b/webapp/app/partials/projects/projects.html
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/app/partials/tables/source_metadata.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/source_metadata.html b/webapp/app/partials/tables/source_metadata.html
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/bower.json
----------------------------------------------------------------------
diff --git a/webapp/bower.json b/webapp/bower.json
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/59411d92/webapp/grunt.json
----------------------------------------------------------------------
diff --git a/webapp/grunt.json b/webapp/grunt.json
old mode 100755
new mode 100644
[31/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dda5ee4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dda5ee4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dda5ee4f
Branch: refs/heads/inverted-index
Commit: dda5ee4f213c5df73d3791e1444b53cfe191e4ff
Parents: c714097
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 15:47:00 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 15:47:00 2015 +0800
----------------------------------------------------------------------
bin/stop-kylin.sh | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dda5ee4f/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
index 4ac8b7c..c7a61b2 100755
--- a/bin/stop-kylin.sh
+++ b/bin/stop-kylin.sh
@@ -1,12 +1,18 @@
#!/bin/sh
-pid=`cat ${KYLIN_HOME}/pid || echo "no pid found";exit 1`
-if [ ! "$pid" = "" ]
+if [ ! -f "${KYLIN_HOME}/pid" ]
then
echo "no pid found"
exit 1
+fi
+
+pid=`cat ${KYLIN_HOME}/pid`
+if [ "$pid" = "" ]
+then
+ echo "pid is empty"
+ exit 1
else
- echo "stopping pid:$pid"
+ echo "stopping kylin:$pid"
kill $pid
fi
rm ${KYLIN_HOME}/pid
[27/50] incubator-kylin git commit: Remove corporate info
Posted by li...@apache.org.
Remove corporate info
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/40ff1cda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/40ff1cda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/40ff1cda
Branch: refs/heads/inverted-index
Commit: 40ff1cda4b6589aa5ff68541ae670f7e42e5a132
Parents: 09f4bc3
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Feb 11 15:43:39 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Feb 11 15:43:39 2015 +0800
----------------------------------------------------------------------
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
jdbc/pom.xml | 5 -----
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
4 files changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40ff1cda/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict b/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict
deleted file mode 100644
index 36da380..0000000
Binary files a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40ff1cda/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict b/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict
deleted file mode 100644
index 5c9af56..0000000
Binary files a/examples/test_case_data/localmeta/dict/PREDEFINED/date(yyyy-mm-dd)/f5e85644-db92-42b5-9ad5-240ab227d7b0.dict and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40ff1cda/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index a1dce3c..606e340 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -22,11 +22,6 @@
<artifactId>kylin-jdbc</artifactId>
<packaging>jar</packaging>
- <!-- More project information. -->
- <name>Kylin:Jdbc</name>
- <description>kylin jdbc on optiq avatica</description>
- <url>https://github.scm.corp.ebay.com/BIPlatform/Kylin</url>
-
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40ff1cda/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
index 4fd64e5..a92da92 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/hbase/TestHbaseClient.java
@@ -87,7 +87,7 @@ public class TestHbaseClient {
foo(3, 0);
Configuration conf = HBaseConfiguration.create();
- conf.set("hbase.zookeeper.quorum", "yadesk00.corp.ebay.com");
+ conf.set("hbase.zookeeper.quorum", "hbase_host");
conf.set("zookeeper.znode.parent", "/hbase-unsecure");
HTable table = new HTable(conf, "test1");
[05/50] incubator-kylin git commit: Merge pull request #421 from
janzhongi/inverted-index
Posted by li...@apache.org.
Merge pull request #421 from janzhongi/inverted-index
update bower.json in webapp
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/15535300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/15535300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/15535300
Branch: refs/heads/inverted-index
Commit: 15535300eff4d8c610d794f42631e558a1b67de4
Parents: 7eb78db e0f69a5
Author: Zhong,Jian <ji...@ebay.com>
Authored: Tue Feb 10 16:14:35 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Tue Feb 10 16:14:35 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/page.js | 1 -
webapp/app/js/controllers/projects.js | 3 +-
webapp/app/js/controllers/sourceMeta.js | 13 ++------
webapp/app/js/model/projectModel.js | 2 ++
webapp/app/js/model/tableModel.js | 7 +++-
webapp/app/partials/cubes/cube_json_edit.html | 2 +-
webapp/app/partials/jobs/job_submit.html | 38 +++++++++++-----------
webapp/bower.json | 5 ++-
8 files changed, 37 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
[47/50] incubator-kylin git commit: merge website content from staging
Posted by li...@apache.org.
merge website content from staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1ce77a52
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1ce77a52
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1ce77a52
Branch: refs/heads/inverted-index
Commit: 1ce77a52e1ac2fa3cd13c20b53329c2ad89b3ad0
Parents: a864ba2
Author: lukehan <lu...@apache.org>
Authored: Thu Feb 12 13:09:25 2015 +0800
Committer: lukehan <lu...@apache.org>
Committed: Thu Feb 12 13:09:25 2015 +0800
----------------------------------------------------------------------
docs/website/_includes/footer.html | 14 +--
docs/website/_includes/head.cn.html | 42 +++++++++
docs/website/_includes/header.cn.html | 39 +++++++++
docs/website/_includes/header.html | 21 +++--
docs/website/_layouts/default-cn.html | 23 +++++
docs/website/_layouts/post.html | 11 ++-
.../_posts/2015-01-25-introduce-data-model.md | 40 +++++++++
docs/website/about/index.md | 77 +----------------
docs/website/assets/css/styles.css | 32 ++++---
docs/website/blog/index.html | 21 ++++-
docs/website/community/index.html | 8 +-
docs/website/css/main.scss | 52 +++++++++++
docs/website/docs/index.html | 2 -
docs/website/index.cn.md | 91 ++++++++++++++++++++
docs/website/index.md | 91 ++++++++++++++++++++
15 files changed, 448 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_includes/footer.html
----------------------------------------------------------------------
diff --git a/docs/website/_includes/footer.html b/docs/website/_includes/footer.html
index b30acb9..8561e95 100644
--- a/docs/website/_includes/footer.html
+++ b/docs/website/_includes/footer.html
@@ -3,16 +3,10 @@
<div class="row">
<div class="col-md-12 widget" >
<div class="widget-body" style="text-align:center">
- <ul class="icons">
- <li><a href="https://www.linkedin.com/groups/KylinOLAP-6785709?home=&gid=6785709&trk=anet_ug_hm" target="_blank" class="fa fa-linkedin fa-lg"></a></li>
- <li><a href="https://twitter.com/debashis_saha/status/519612003443146752" target="_blank" class="fa fa-twitter fa-lg"></a></li>
- <li><a href="https://www.facebook.com/kylinio?skip_nax_wizard=true&ref_type=logout_gear" target="_blank" class="fa fa-facebook fa-lg"></a></li>
- <li><a href="https://plus.google.com/communities/101138166999680123468" target="_blank" class="fa fa-google-plus fa-lg"></a></li>
- <li><a href="https://github.com/KylinOLAP/Kylin" target="_blank" class="fa fa-github-alt fa-lg"></a></li>
- <!-- <li><a href="#" class="fa fa-weibo fa-lg"></a></li>
- <li><a href="#" class="fa fa-weixin fa-lg"></a></li>-->
- </ul>
- <p style="text-align:center" > Copyright ©2014 <a href="http://kylin.io">Kylin.io</a> All Rights Reserved. | From <a href="http://www.ebayinc.com/">eBay Inc.</a> |<a href="#">back to top</a><br>
+ <div>
+ Apache Kylin is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
+ </div>
+ <p style="text-align:center" > Apache Kylin is distributed under the <a href="http://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License v2.0. </a>
</p>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_includes/head.cn.html
----------------------------------------------------------------------
diff --git a/docs/website/_includes/head.cn.html b/docs/website/_includes/head.cn.html
new file mode 100644
index 0000000..ae98dee
--- /dev/null
+++ b/docs/website/_includes/head.cn.html
@@ -0,0 +1,42 @@
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Kylin.IO | {% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %}</title>
+ <meta name="description" content="{% if page.excerpt %}{{ page.excerpt | strip_html | strip_newlines | truncate: 160 }}{% else %}{{ site.description }}{% endif %}">
+ <meta name="author" content="Kylin.IO">
+ <link rel="shortcut icon" href="fav.png" type="image/png">
+
+
+
+<link rel="stylesheet" href="{{ "/assets/css/animate.css"| prepend: site.baseurl }}">
+<!-- Bootstrap -->
+<link rel="stylesheet" href="{{ "/assets/css/bootstrap.min.css"| prepend: site.baseurl }}">
+
+<!-- Fonts -->
+<link rel="stylesheet" href="http://fonts.googleapis.com/css?family=Alice|Open+Sans:400,300,700">
+
+<!-- Icons -->
+<link rel="stylesheet" href="{{ "/assets/css/font-awesome.min.css"| prepend: site.baseurl }}">
+
+ <!-- Custom styles -->
+ <link rel="stylesheet" href="{{ "/assets/css/styles.css" | prepend: site.baseurl }}">
+
+ <link rel="canonical" href="{{ page.url | replace:'index.html','' | prepend: site.baseurl | prepend: site.url }}">
+ <link rel="alternate" type="application/rss+xml" title="{{ site.title }}" href="{{ "/feed.xml" | prepend: site.baseurl | prepend: site.url }}" />
+
+<!--[if lt IE 9]> <script src="assets/js/html5shiv.js"></script> <![endif]-->
+<script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-55534813-1', 'auto');
+ ga('send', 'pageview');
+
+
+</script>
+
+</head>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_includes/header.cn.html
----------------------------------------------------------------------
diff --git a/docs/website/_includes/header.cn.html b/docs/website/_includes/header.cn.html
new file mode 100644
index 0000000..63cd04a
--- /dev/null
+++ b/docs/website/_includes/header.cn.html
@@ -0,0 +1,39 @@
+<header id="header" >
+ <div id="head" class="parallax" parallax-speed="3" >
+ <div id="logo" class="text-center"> <img class="img-circle" id="circlelogo" src="{{ "/assets/images/kylin_logo.jpg"| prepend: site.baseurl }}"> <span class="title" >Apache Kylin</span> <span class="tagline">Extreme OLAP Engine for Big Data</span>
+ </div>
+ </div>
+
+ <!-- Main Menu -->
+ <nav class="navbar navbar-default" role="navigation" id="nav-wrapper">
+ <div class="container-fluid" id="nav">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
+ </div>
+
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/cn"><b>首页</b></a></li>
+ <li><a href="/docs" >文档</a></li>
+ <li><a href="/community" >社区</a></li>
+ <li><a href="/blog">博客</li>
+ <li><a href="/about" >关于</a></li>
+ <li><a href="/" >English</a></li>
+ <li><a href="https://twitter.com/apachekylin" target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" ></a></li>
+ <li><a href="https://github.com/apache/incubator-kylin" target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/incubator-kylin" ></a></li>
+ <li><a href="https://www.facebook.com/kylinio" target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" ></a></li>
+ </ul>
+
+
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container-fluid -->
+</nav>
+ </header>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_includes/header.html
----------------------------------------------------------------------
diff --git a/docs/website/_includes/header.html b/docs/website/_includes/header.html
index 9e792b5..1745744 100644
--- a/docs/website/_includes/header.html
+++ b/docs/website/_includes/header.html
@@ -1,16 +1,17 @@
<header id="header" >
+
<div id="head" class="parallax" parallax-speed="3" >
- <div id="logo" class="text-center"> <img class="img-circle" id="circlelogo" src="{{ "/assets/images/kylin_logo.jpg"| prepend: site.baseurl }}"> <span class="title" >Apache Kylin</span> <span class="tagline">Extreme OLAP Engine for Big Data<br>
- </span>
- <!--
- <div id="download"><a href="https://github.com/KylinOLAP/Kylin/releases">DOWNLOAD</a></div>
- -->
- </div>
+ <div id="logo" class="text-center"> <img class="img-circle" id="circlelogo" src="{{ "/assets/images/kylin_logo.jpg"| prepend: site.baseurl }}"> <span class="title" >Apache Kylin</span> <span class="tagline">Extreme OLAP Engine for Big Data</span>
+ </div>
</div>
+
<!-- Main Menu -->
<nav class="navbar navbar-default" role="navigation" id="nav-wrapper">
<div class="container-fluid" id="nav">
+ <!--
+ <img class="img-circle" width="40px" height="40px" id="circlelogo" src="{{ "/assets/images/kylin_logo.jpg"| prepend: site.baseurl }}">
+ -->
<!-- Brand and toggle get grouped for better mobile display -->
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
@@ -30,9 +31,11 @@
<li><a href="/community" >Community</a></li>
<li><a href="/blog">Blog</li>
<li><a href="/about" >About</a></li>
- </ul>
-
-
+ <li><a href="/cn" >中文版</a></li>
+ <li><a href="https://twitter.com/apachekylin" target="_blank" class="fa fa-twitter fa-lg" title="Twitter: @ApacheKylin" ></a></li>
+ <li><a href="https://github.com/apache/incubator-kylin" target="_blank" class="fa fa-github-alt fa-lg" title="Github: apache/incubator-kylin" ></a></li>
+ <li><a href="https://www.facebook.com/kylinio" target="_blank" class="fa fa-facebook fa-lg" title="Facebook: kylin.io" ></a></li>
+ </ul>
</div><!-- /.navbar-collapse -->
</div><!-- /.container-fluid -->
</nav>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_layouts/default-cn.html
----------------------------------------------------------------------
diff --git a/docs/website/_layouts/default-cn.html b/docs/website/_layouts/default-cn.html
new file mode 100644
index 0000000..1adae5b
--- /dev/null
+++ b/docs/website/_layouts/default-cn.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html>
+
+ {% include head.cn.html %}
+
+ <body>
+
+ {% include header.cn.html %}
+
+ <div class="page-content">
+ <div class="wrapper">
+ {{ content }}
+ </div>
+ </div>
+
+ {% include footer.html %}
+
+ <script src="/assets/js/jquery-1.9.1.min.js"></script>
+ <script src="/assets/js/bootstrap.min.js"></script>
+ <script src="/assets/js/main.js"></script>
+ </body>
+
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_layouts/post.html
----------------------------------------------------------------------
diff --git a/docs/website/_layouts/post.html b/docs/website/_layouts/post.html
index a2b4e52..0574e66 100644
--- a/docs/website/_layouts/post.html
+++ b/docs/website/_layouts/post.html
@@ -1,15 +1,20 @@
---
layout: default
---
-<div class="post">
+
+<div class="post" style=" padding:2em 4em 4em 4em">
<header class="post-header">
<h1 class="post-title">{{ page.title }}</h1>
- <p class="post-meta">{{ page.date | date: "%b %-d, %Y" }}{% if page.author %} • {{ page.author }}{% endif %}{% if page.meta %} • {{ page.meta }}{% endif %}</p>
+ <p class="post-meta" >{{ page.date | date: "%b %-d, %Y" }}{% if page.author %} • {{ page.author }}{% endif %}{% if page.meta %} • {{ page.meta }}{% endif %}</p>
</header>
- <article class="post-content">
+ <article class="post-content" >
{{ content }}
</article>
</div>
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/_posts/2015-01-25-introduce-data-model.md
----------------------------------------------------------------------
diff --git a/docs/website/_posts/2015-01-25-introduce-data-model.md b/docs/website/_posts/2015-01-25-introduce-data-model.md
new file mode 100644
index 0000000..2016207
--- /dev/null
+++ b/docs/website/_posts/2015-01-25-introduce-data-model.md
@@ -0,0 +1,40 @@
+---
+layout: post
+title: "Introduce Data Model of Cube Designer"
+date: 2015-01-25 22:28:00
+author: Luke Han
+categories: blog
+---
+
+### Background
+In previous version (before v0.6.4), Kylin introduced a GUI tool called Cube Designer for user (we called this role as __Cube Modeler__) to architect OLAP Cube with dimensions, measures and other settings. It works well for most of the features but still not user friendly yet:
+
+1. A user has to add dimension one by one, considering there are 20+ even 50+ dimensions, the entire process is really boring.
+2. Each dimension requires define join condition between fact table and lookup table which even already be defined in previous dimensions many times.
+3. Less validation check, especially for Hierarchy and Derived dimension, there are many exceptions in further steps which blocked many people to save the cube definition without any idea about the issue.
+4. Save/Next buttons are confusing user to click which one for real next step or just save current dimension settings
+
+### Data Model of Cube Designer
+With the feedback from our internal users and external community, we have came up one idea and would like to introduce a new concept (widely known in Data Warehouse and Business Intelligence domain): Data Model: a data model organises data elements and standardises how the data elements relate to one another.[Wikipedia](http://en.wikipedia.org/wiki/Data_model). In Kylin, it using [Star Schema](http://en.wikipedia.org/wiki/Star_schema) as Data Model, which is the simplest style of data warehouse schema. The star schema consists of a few "fact tables" (possibly only one, justifying the name) referencing any number of "dimension tables". It actually already there behind dimensions and measures and now just come to first step to define the relationship between different tables before create each dimension.
+Now (after v0.6.4), to create a cube will follow below steps:
+
+1. Define data model first: pick up one fact table and then add other lookup tables (with their join conditions). The data mode must be presents as Star Schema.
+2. Then add dimensions, since all join conditions already presented in data model, each dimension could be more easy to create, just need to know what's kind of type: normal, hierarchy and derived (will have another blog to introduce them). There's also one helper called _Auto Generator_ to help generate many dimensions within simple clicks.
+3. Then define measures and others as previous cube designer did
+
+### Benefits
+1. A data model is very easy to communicate between different roles and teams. Most of cases it just mapping to real database table relationship, like from Hive tables
+2. More easy to create dimensions and measures based on the data model
+3. Friendly error message with enhanced validation check when save cube
+
+
+### What's Next
+After this refactor, Kylin is now be able to introduce more powerful features, the major idea is to using different storages to serve same data model:
+
+* __Logical Model__: A Data Model presents logical data structure like Star Schema beyond data tables with more business meaning
+* __Physical Model__: define how the underlying data to be stored in persistent system, like HBase. There are already two of them: MOLAP (current Kylin version) and InvertedIndex (coming with 0.7.x release). And it also easy to extend to support others without change the Logical Model.
+* A new GUI of Cube Designer to support above is on the way.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/about/index.md
----------------------------------------------------------------------
diff --git a/docs/website/about/index.md b/docs/website/about/index.md
index d165f1f..100a57b 100644
--- a/docs/website/about/index.md
+++ b/docs/website/about/index.md
@@ -13,7 +13,9 @@ title: About
<div class="row">
<div class="col-sm-12 col-md-12">
<div >
- <p class="aboutkylin" style="font-size:1.2em">Kylin, which is an OLAP Engine for Hadoop, has been developed by eBay. Now it becomes available to all as an open source project.</p>
+ <p class="aboutkylin" style="font-size:1.2em">Apache Kylin, which is a distributed and scalable OLAP engine built on Hadoop to support
+extremely large datasets, developed and contributed by <a href="http://www.ebayinc.com/" target="_blank">eBay Inc</a> to open source community on Oct 1, 2014 and has been Apache Incubator Project since Nov 25, 2014. Refer to announcement on eBay Tech Blog for more detail: <a href="http://www.ebaytechblog.com/2014/10/20/announcing-kylin-extreme-olap-engine-for-big-data" target="_blank">Announcing Kylin: Extreme OLAP Engine for Big Data</a> </p>
+
</div>
</div>
@@ -23,79 +25,6 @@ title: About
</header>
</section>
-
-
- <div id="second" class=" main" style="background-color:#efefef;" >
- <header style="background-color:#efefef;" >
- <div class="container">
- <h4 class="section-title"><span>Kylin Contributors</span></h4>
- <div class="row" style="margin-top:-20px;" id="contributors">
- <p class="core_team">Kylin is an open source project from eBay, migrated from internal github repo into Github repo without history information:-(</p>
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Luke Han <a href="https://github.com/lukehan" target="_blank">(lukehan)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee (Project Lead)</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- Releases</span><br/>
-<span class="maintains">- Roadmap</span></p>
-
- </div>
-
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Jiang Xu <a href="https://github.com/jiangxuchina" target="_blank">(jiangxuchina)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee (Cheif Architect)</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- Architecture & Design</span><br/>
-<span class="maintains">- Query Engine</span></p>
-
- </div>
-
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Yang Li <a href="https://github.com/liyang-gmt8" target="_blank">(liyang-gmt8)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee (Tech Lead)</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- Query Engine</span><br/>
-<span class="maintains">- Job Engine</span><br/>
-<span class="maintains">- Storage Engine</span></p>
-
- </div>
-
- </div>
-
- <div class="row" id="contributors2">
-
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Yi Song <a href="https://github.com/songyi10011001" target="_blank">(songyi10011001)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- Job Engine</span><br/>
-<span class="maintains">- Build System</span></p>
-
- </div>
-
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Hongbin Ma <a href="https://github.com/binmahone" target="_blank">(binmahone)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- ODBC Driver</span><br/>
-<span class="maintains">- Metadata Engine</span></p>
-
- </div>
-
- <div class="col-sm-4 col-md-4">
- <p class="normal_text people_name">Xiaodong Duo <a href="https://github.com/xduo" target="_blank">(xduo)</a></p>
- <p><span class="people_intro">Full-Time eBay Employee (Tech Lead)</span><br/><span class="maintain">Maintains:</span><br/>
-<span class="maintains">- Job Engine</span><br/>
-<span class="maintains">- REST Server</span><br/>
-<span class="maintains">- JDBC Driver</span></p>
-
- </div>
-
- </div>
-
-
- <p class="normal_text">Participant: <p>
- <p class="normal_text participant">Rui Feng <a href="https://github.com/fengrui129" target="_blank">(fengrui129)</a></p>
-
- </div>
- <!-- /container -->
-
- </header>
- </div>
<!-- / section -->
</div>
<!-- /container -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/assets/css/styles.css
----------------------------------------------------------------------
diff --git a/docs/website/assets/css/styles.css b/docs/website/assets/css/styles.css
index 12ebef4..e2e080e 100644
--- a/docs/website/assets/css/styles.css
+++ b/docs/website/assets/css/styles.css
@@ -118,7 +118,7 @@ a:hover, a:focus {
background: #f4f4f4 url(../images/data.png) top center;
background-size: cover;
color: #7C7C7C;
- height: 400px
+ height: 200px;
}
#head .title {
font-family: 'Open sans';
@@ -136,8 +136,8 @@ a:hover, a:focus {
}
#head img.img-circle {
display: block;
- width: 140px;
- height: 140px;
+ width: 100px;
+ height: 100px;
overflow: hidden;
border: 9px solid rgba(0, 0, 0, 0.05);
margin: 0 auto;
@@ -161,10 +161,11 @@ a:hover, a:focus {
color: #5E5E5E;
}
#logo {
- padding-top: 50px;
+ padding-top: 0px;
+ padding-left: 10px;
}
.home #head .title {
- font-size: 36px;
+ font-size: 18px;
color: #fb8d08;/*color:#146df3;*/
}
@@ -203,7 +204,13 @@ a:hover, a:focus {
**********************************************************************/
.navbar {
- border-width: 1px 0;
+ /*
+ background: #f4f4f4 url(../images/data.png) top center;
+ background-size: cover;
+ color: #7C7C7C;
+ height: 100px;
+*/
+ border-width: 1px 0;
-webkit-border-radius: 0;
-webkit-background-clip: padding-box;
-moz-border-radius: 0;
@@ -211,6 +218,7 @@ a:hover, a:focus {
border-radius: 0;
background-clip: padding-box;
width: 100%;
+
}
.navbar.stick {
position: fixed;
@@ -227,19 +235,19 @@ a:hover, a:focus {
background-clip: padding-box;
font-family: 'Open sans';
font-weight: 400;
- text-transform: uppercase;
+ /* text-transform: uppercase;*/
}
.navbar-nav {
float: none;
margin: 0 auto;
- text-align: center;
+ text-align: right;
}
.navbar-nav > li {
float: none;
display: inline-block;
}
.navbar-nav > li > a {
- padding: 20px 30px;
+ padding: 10px 12px;
}
.navbar-default {
background-color: #ffffff;
@@ -553,9 +561,9 @@ ol.style1 {
margin: 0 0 20px;
}
#underfooter {
- background: #191919;
+ background: #696969;
padding: 15px 0;
- color: #777;
+ color: #EEEEEE;
font-size: 12px;
}
#underfooter a {
@@ -580,7 +588,7 @@ ol.style1 {
background-clip: padding-box;
font-family: 'Open sans';
font-weight: 400;
- text-transform: uppercase;
+ /* text-transform: uppercase;*/
max-height: 340px;
padding-right: 15px;
padding-left: 15px;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/blog/index.html
----------------------------------------------------------------------
diff --git a/docs/website/blog/index.html b/docs/website/blog/index.html
index 0fc5a3b..6db414f 100644
--- a/docs/website/blog/index.html
+++ b/docs/website/blog/index.html
@@ -10,14 +10,27 @@ title: Blog
<div class="container" >
<h4 class="section-title"><span> Kylin Technical Blog </span></h4>
<!-- second-->
- <div class="row">
- <div class="col-sm-12 col-md-12">
+ <div id="content-container" class="animated fadeIn">
<div >
+ <!--
<p class="aboutkylin" style="font-size:1.2em">Comming Soon...</p>
-
+ -->
+ <ul class="post-list">
+ {% for post in site.posts %}
+ <li>
+ <!--
+ <span align="left" class="content-header">{{ post.date | date: "%b %-d, %Y" }}</span>
+ -->
+ <h2 align="left">
+ <a class="post-link" href="{{ post.url | prepend: site.baseurl }}">{{ post.title }}</a>
+ </h2>
+ </li>
+ {% endfor %}
+ </ul>
+
+ <p class="rss-subscribe">subscribe <a href="{{ "/feed.xml" | prepend: site.baseurl }}">via RSS</a></p>
</div>
</div>
- </div>
</div>
<!-- /container -->
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/community/index.html
----------------------------------------------------------------------
diff --git a/docs/website/community/index.html b/docs/website/community/index.html
index 9b2bbe7..c0452d8 100644
--- a/docs/website/community/index.html
+++ b/docs/website/community/index.html
@@ -16,6 +16,11 @@ title: Community
+
+ <p><a href="http://strataconf.com/big-data-conference-uk-2015/public/schedule/detail/40029" target="_blank">Strata+Hadoop World 2015, London </a>
+ <br/>Date - Wednesday, May 6, 2015 <br/>
+ Kylin will be present at Strata+Hadooop World</p>
+
<p><a href="http://bdtc2014.hadooper.cn/m/zone/bdtc_2014" target="_blank">Big Data Technology Conference, Beijing </a>
<br/>Date - Sunday, December 14, 2014 <br/>
Kylin be presented by Luke Han, Sr. Product Manager of Kylin</p>
@@ -33,8 +38,7 @@ title: Community
<div class="col-sm-4 col-md-4">
<p class="big_text">DISCUSSION</p>
- <p>Developement and User maililng list: <a href="mailto:dev@kylin.incubator.apache.org" target="_blank"> dev@kylin.incubator.apache.org</a></p>
- <p>The <a href="https://groups.google.com/forum/#!forum/kylin-olap" target="_blank">Google Group </a> is the place for discussions of Kylin features</p>
+ <p>Developement and User maililng list: <a href="mailto:dev@kylin.incubator.apache.org" target="_blank"> dev@kylin.incubator.apache.org</a></p>
<p>The official Kylin Twitter account: <a href="https://twitter.com/ApacheKylin" target="_blank"> @ApacheKylin</a></p>
</div>
<div class="col-sm-4 col-md-4">
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/css/main.scss
----------------------------------------------------------------------
diff --git a/docs/website/css/main.scss b/docs/website/css/main.scss
new file mode 100755
index 0000000..beee4e3
--- /dev/null
+++ b/docs/website/css/main.scss
@@ -0,0 +1,52 @@
+---
+# Only the main Sass file needs front matter (the dashes are enough)
+---
+@charset "utf-8";
+
+
+
+// Our variables
+$base-font-family: Helvetica, Arial, sans-serif;
+$base-font-size: 16px;
+$small-font-size: $base-font-size * 0.875;
+$base-line-height: 1.5;
+
+$spacing-unit: 30px;
+
+$text-color: #111;
+$background-color: #fdfdfd;
+$brand-color: #2a7ae2;
+
+$grey-color: #828282;
+$grey-color-light: lighten($grey-color, 40%);
+$grey-color-dark: darken($grey-color, 25%);
+
+// Width of the content area
+$content-width: 800px;
+
+$on-palm: 600px;
+$on-laptop: 800px;
+
+
+
+// Using media queries with like this:
+// @include media-query($on-palm) {
+// .wrapper {
+// padding-right: $spacing-unit / 2;
+// padding-left: $spacing-unit / 2;
+// }
+// }
+@mixin media-query($device) {
+ @media screen and (max-width: $device) {
+ @content;
+ }
+}
+
+
+
+// Import partials from `sass_dir` (defaults to `_sass`)
+@import
+ "base",
+ "layout",
+ "syntax-highlighting"
+;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/docs/index.html
----------------------------------------------------------------------
diff --git a/docs/website/docs/index.html b/docs/website/docs/index.html
index 5057a13..599fd75 100644
--- a/docs/website/docs/index.html
+++ b/docs/website/docs/index.html
@@ -2,8 +2,6 @@
layout: default
title: Docs
---
-
-
<main id="main" >
<div class="container" >
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/index.cn.md
----------------------------------------------------------------------
diff --git a/docs/website/index.cn.md b/docs/website/index.cn.md
new file mode 100644
index 0000000..b0d19ee
--- /dev/null
+++ b/docs/website/index.cn.md
@@ -0,0 +1,91 @@
+---
+layout: default-cn
+title: 首页
+---
+
+
+<main id="main" >
+ <div class="container" >
+ <div id="zero" class=" main" >
+ <header style=" padding:2em 0 4em 0">
+ <div class="container" >
+
+ <h4 class="section-title"><span>Apache Kylin 概览</span></h4>
+ <div class="row" style="margin-top:-20px;">
+ <div class="col-sm-12 col-md-12">
+ <p class="title_text"> Kylin 于2014年11月25日被接受会Apache孵化器项目</p>
+ <p class="title_text"> Apache Kylin 是由eBay开源的分布式分析引擎，提供Hadoop之上的SQL查询接口及多维分析（OLAP）能力以支持超大规模数据</p>
+ <img id="diagram" src="{{ "/assets/images/kylin_diagram.png"| prepend: site.baseurl }}"> </div>
+ </div>
+ </div>
+ <!-- /container -->
+
+ </header>
+ </div>
+ <!-- / section -->
+ </div>
+ <!-- /container -->
+
+ <section id="second" class="main">
+ <header style="background-color:#efefef;">
+ <div class="container" >
+ <h4 class="section-title"><span> Kylin是什么? </span></h4>
+ <!-- second-->
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div align="left">
+ <p> <b>- 可扩展超快OLAP引擎: </b><br/>
+ <div class="indent">Kylin是为减少在Hadoop上百亿规模数据查询延迟而设计</div>
+ </p>
+ <p> <b>- Hadoop ANSI SQL 接口: </b><br/>
+ <div class="indent">Kylin为Hadoop提供标准SQL支持大部分查询功能</div>
+ </p>
+ <p> <b>- 交互式查询能力: </b><br/>
+ <div class="indent">通过Kylin，用户可以与Hadoop数据进行亚秒级交互，在同样的数据集上提供比Hive更好的性能</div>
+ </p>
+ <p> <b>- 多维立方体（MOLAP Cube）:</b><br/>
+ <div class="indent">用户能够在Kylin里为百亿以上数据集定义数据模型并构建立方体</div>
+ </p>
+ <p> <b>- 与BI工具无缝整合:</b><br/>
+ <div class="indent">Kylin提供与BI工具，如Tableau，的整合能力，即将提供对其他工具的整合</div>
+ </p>
+ <p> <b>- 其他特性:</b> <br/>
+ <div class="indent">- Job管理与监控 <br/>
+ - 压缩与编码 <br/>
+ - 增量更新 <br/>
+ - 利用HBase Coprocessor<br/>
+ - 基于HyperLogLog的Dinstinc Count近似算法 <br/>
+ - 友好的web界面以管理，监控和使用立方体 <br/>
+ - 项目及立方体级别的访问控制安全<br/>
+ - 支持LDAP </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- /container -->
+
+ </header>
+ </section>
+
+ <!-- second -->
+ <section id="first" class="main">
+ <header>
+ <div class="container" >
+ <h4 class="section-title"><span>Kylin 生态圈</span></h4>
+ <div class="row">
+ <div class="col-sm-7 col-md-7">
+ <p> </p>
+ <p><b>Kylin 核心:</b> Kylin OLAP引擎基础框架，包括元数据（Metadata）引擎，查询引擎，Job引擎及存储引擎等，同时包括REST服务器以响应客户端请求</p>
+ <p><b>扩展:</b> 支持额外功能和特性的插件</p>
+ <p><b>整合:</b> 与调度系统，ETL，监控等生命周期管理系统的整合</p>
+ <p><b>用户界面:</b> 在Kylin核心之上扩展的第三方用户界面</p>
+ <p><b>驱动:</b> ODBC 和 JDBC 驱动以支持不同的工具和产品，比如Tableau</p>
+ </div>
+ <div class="col-sm-5 col-md-5"> </div>
+ </div>
+ <!-- /container -->
+ </div>
+ </header>
+ </section>
+</main>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1ce77a52/docs/website/index.md
----------------------------------------------------------------------
diff --git a/docs/website/index.md b/docs/website/index.md
new file mode 100644
index 0000000..f18a7c3
--- /dev/null
+++ b/docs/website/index.md
@@ -0,0 +1,91 @@
+---
+layout: default
+title: Home
+---
+
+
+<main id="main" >
+ <div class="container" >
+ <div id="zero" class=" main" >
+ <header style=" padding:2em 0 4em 0">
+ <div class="container" >
+
+ <h4 class="section-title"><span>Apache Kylin Overview</span></h4>
+ <div class="row" style="margin-top:-20px;">
+ <div class="col-sm-12 col-md-12">
+ <p class="title_text"> Kylin has been accepted as Apache Incubator Project on Nov 25, 2014.</p>
+ <p class="title_text"> Apache Kylin is an open source Distributed Analytics Engine from eBay Inc. that provides SQL interface and multi-dimensional analysis (OLAP) on Hadoop supporting extremely large datasets</p>
+ <img id="diagram" src="assets/images/kylin_diagram.png"> </div>
+ </div>
+ </div>
+ <!-- /container -->
+
+ </header>
+ </div>
+ <!-- / section -->
+ </div>
+ <!-- /container -->
+
+ <section id="second" class="main">
+ <header style="background-color:#efefef;">
+ <div class="container" >
+ <h4 class="section-title"><span> What is Kylin? </span></h4>
+ <!-- second-->
+ <div class="row">
+ <div class="col-sm-12 col-md-12">
+ <div align="left">
+ <p> <b>- Extremely Fast OLAP Engine at Scale: </b><br/>
+ <div class="indent">Kylin is designed to reduce query latency on Hadoop for 10+ billions of rows of data</div>
+ </p>
+ <p> <b>- ANSI SQL Interface on Hadoop: </b><br/>
+ <div class="indent">Kylin offers ANSI SQL on Hadoop and supports most ANSI SQL query functions</div>
+ </p>
+ <p> <b>- Interactive Query Capability: </b><br/>
+ <div class="indent">Users can interact with Hadoop data via Kylin at sub-second latency, better than Hive queries for the same dataset</div>
+ </p>
+ <p> <b>- MOLAP Cube:</b><br/>
+ <div class="indent">User can define a data model and pre-build in Kylin with more than 10+ billions of raw data records</div>
+ </p>
+ <p> <b>- Seamless Integration with BI Tools:</b><br/>
+ <div class="indent">Kylin currently offers integration capability with BI Tools like Tableau. Integration with Microstrategy and Excel is coming soon</div>
+ </p>
+ <p> <b>- Other Highlights:</b> <br/>
+ <div class="indent">- Job Management and Monitoring <br/>
+ - Compression and Encoding Support <br/>
+ - Incremental Refresh of Cubes <br/>
+ - Leverage HBase Coprocessor for query latency <br/>
+ - Approximate Query Capability for distinct Count (HyperLogLog) <br/>
+ - Easy Web interface to manage, build, monitor and query cubes <br/>
+ - Security capability to set ACL at Cube/Project Level <br/>
+ - Support LDAP Integration </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- /container -->
+
+ </header>
+ </section>
+
+ <!-- second -->
+ <section id="first" class="main">
+ <header>
+ <div class="container" >
+ <h4 class="section-title"><span>Kylin Ecosystem</span></h4>
+ <div class="row">
+ <div class="col-sm-7 col-md-7">
+ <p> </p>
+ <p><b>Kylin Core:</b> Fundamental framework of Kylin OLAP Engine comprises of Metadata Engine, Query Engine, Job Engine and Storage Engine to run the entire stack. It also includes a REST Server to service client requests</p>
+ <p><b>Extensions:</b> Plugins to support additional functions and features </p>
+ <p><b>Integration:</b> Lifecycle Management Support to integrate with Job Scheduler, ETL, Monitoring and Alerting Systems </p>
+ <p><b>User Interface:</b> Allows third party users to build customized user-interface atop Kylin core</p>
+ <p><b>Drivers:</b> ODBC and JDBC drivers to support different tools and products, such as Tableau</p>
+ </div>
+ <div class="col-sm-5 col-md-5"> <img id="core" src="assets/images/core.png"> </div>
+ </div>
+ <!-- /container -->
+ </div>
+ </header>
+ </section>
+</main>
[03/50] incubator-kylin git commit: Update test case
Posted by li...@apache.org.
Update test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7eb78db0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7eb78db0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7eb78db0
Branch: refs/heads/inverted-index
Commit: 7eb78db0e5b48a5680721ea9ef839a2c04cfcd56
Parents: e590e3e
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Feb 10 14:47:24 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Feb 10 14:47:24 2015 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/jdbc/JDBCDriverTest.java | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7eb78db0/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
index 4f10868..9aa8315 100644
--- a/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
+++ b/server/src/test/java/org/apache/kylin/jdbc/JDBCDriverTest.java
@@ -25,12 +25,14 @@ public class JDBCDriverTest extends HBaseMetadataTestCase {
public static void beforeClass() throws Exception {
previousSpringProfile = System.getProperty(SPRING_PROFILE_PROPERTY);
System.setProperty(SPRING_PROFILE_PROPERTY, "testing");
+ staticCreateTestMetadata();
startJetty();
}
@AfterClass
public static void afterClass() throws Exception {
stopJetty();
+ staticCleanupTestMetadata();
if (previousSpringProfile == null) {
System.clearProperty(SPRING_PROFILE_PROPERTY);
} else {
@@ -38,16 +40,6 @@ public class JDBCDriverTest extends HBaseMetadataTestCase {
}
}
- @Before
- public void setup() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
protected static void stopJetty() throws Exception {
if (server != null)
server.stop();
[17/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/376941bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/376941bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/376941bd
Branch: refs/heads/inverted-index
Commit: 376941bdc508fbf4de12663ff1dbc32bba948f07
Parents: 681314c
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:49:20 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:49:20 2015 +0800
----------------------------------------------------------------------
script/build.sh | 2 +-
script/package.sh | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/376941bd/script/build.sh
----------------------------------------------------------------------
diff --git a/script/build.sh b/script/build.sh
index 08b2b0a..3014123 100644
--- a/script/build.sh
+++ b/script/build.sh
@@ -11,7 +11,7 @@ mvn clean install -DskipTests
echo 'package front-end'
cd webapp
npm install -g bower
-bower install
+bower --allow-root install
npm install
npm install -g grunt-cli
grunt dev --buildEnv=dev
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/376941bd/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
index 6252690..018491e 100755
--- a/script/package.sh
+++ b/script/package.sh
@@ -26,8 +26,8 @@ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpress
echo "kylin version: ${version}"
export version
-sh script/build.sh || { exit 1; }
-sh script/prepare.sh || { exit 1; }
-sh script/download-tomcat.sh || { exit 1; }
-sh scirpt/compress.sh || { exit 1; }
+sh ${dir}/build.sh || { exit 1; }
+sh ${dir}/prepare.sh || { exit 1; }
+sh ${dir}/download-tomcat.sh || { exit 1; }
+sh ${dir}/compress.sh || { exit 1; }
[30/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8466b53b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8466b53b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8466b53b
Branch: refs/heads/inverted-index
Commit: 8466b53b0fc09074f7b45493083f6111c0a05dab
Parents: 6ef8be0 f460a14
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:45:21 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:45:21 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 2 +-
bin/stop-kylin.sh | 12 ++++++++++--
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
jdbc/pom.xml | 5 -----
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
6 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[04/50] incubator-kylin git commit: updte bower.json
Posted by li...@apache.org.
updte bower.json
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e0f69a50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e0f69a50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e0f69a50
Branch: refs/heads/inverted-index
Commit: e0f69a50cb8c305afc0e2f5c3754dc7c13ad5fa1
Parents: ece2e07
Author: jiazhong <ji...@ebay.com>
Authored: Tue Feb 10 16:13:02 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Feb 10 16:13:02 2015 +0800
----------------------------------------------------------------------
webapp/bower.json | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f69a50/webapp/bower.json
----------------------------------------------------------------------
diff --git a/webapp/bower.json b/webapp/bower.json
index 457b950..31ea559 100755
--- a/webapp/bower.json
+++ b/webapp/bower.json
@@ -25,7 +25,10 @@
"angular-underscore": "~0.5.0",
"angular-ui-sortable": "0.13.1",
"underscore": "~1.7.0",
- "fuelux": "~3.5.1"
+ "fuelux": "~3.5.1",
+ "angular-animate": "1.2",
+ "angular-cookies":"1.2"
+
},
"devDependencies": {
"less.js": "~1.4.0",
[42/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/47263143
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/47263143
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/47263143
Branch: refs/heads/inverted-index
Commit: 47263143a3b577b8989fe8e6dd847d265eee6dfe
Parents: 56ba513 d5a185e
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Feb 12 10:35:55 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Feb 12 10:35:55 2015 +0800
----------------------------------------------------------------------
.gitignore | 0
bin/check-env.sh | 0
bin/start-kylin.sh | 0
bin/stop-kylin.sh | 0
.../common/hll/HyperLogLogPlusCounter.java | 80 +------------------
conf/kylin.properties | 0
.../16d8185c-ee6b-4f8c-a919-756d9809f937.dict | Bin 2515 -> 2630 bytes
.../c12ae49d-9dbe-4a58-b169-19afac317696.dict | Bin 2515 -> 2630 bytes
.../eaed91b0-4182-4ee5-a733-1047a622ee29.dict | Bin 2515 -> 2630 bytes
.../ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict | Bin 2515 -> 2630 bytes
.../7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict | Bin 2940 -> 3105 bytes
.../8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict | Bin 2940 -> 3105 bytes
.../ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict | Bin 2940 -> 3105 bytes
.../eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict | Bin 2940 -> 3105 bytes
.../2602386c-debb-4968-8d2f-b52b8215e385.dict | Bin 1841 -> 2000 bytes
.../4243889f-bc81-4807-a975-7041bbbf35e7.dict | Bin 1841 -> 2000 bytes
.../494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict | Bin 1841 -> 2000 bytes
.../8b4b1c06-fb74-486b-a2ad-74420afebcda.dict | Bin 1841 -> 2000 bytes
.../ac520edd-f9d2-419f-a7de-587bfb97dc81.dict | Bin 1841 -> 2000 bytes
.../0410d2c4-4686-40bc-ba14-170042a2de94.dict | Bin 1498 -> 1537 bytes
.../51ff5e6c-22b3-444e-9915-d376a10f20cb.dict | Bin 1498 -> 1537 bytes
.../a4e57e55-48fc-4f25-a9c8-485deed25925.dict | Bin 1498 -> 1537 bytes
.../aceae914-4246-4251-a0c2-692fe7a300df.dict | Bin 1498 -> 1537 bytes
.../b298089f-9656-4693-b9b2-8fea46f06dd5.dict | Bin 1498 -> 1537 bytes
.../c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict | Bin 1498 -> 1537 bytes
.../0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict | Bin 661 -> 640 bytes
.../14fe66b3-5956-498c-bd93-40182cac5510.dict | Bin 661 -> 640 bytes
.../1d383a36-81b9-4177-a822-04eab3683e5b.dict | Bin 661 -> 640 bytes
.../c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict | Bin 661 -> 640 bytes
.../0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict | Bin 2189 -> 2264 bytes
.../31edf35b-ffca-4f24-8229-f87dc34e3087.dict | Bin 2189 -> 2264 bytes
.../652bd393-678a-4f16-a504-fd8ce1229355.dict | Bin 2189 -> 2264 bytes
.../792eb972-d046-48e6-9428-f6a3aed92fad.dict | Bin 2189 -> 2264 bytes
.../914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict | Bin 2189 -> 2264 bytes
.../d25e554e-deac-4e4a-9289-96f0d121d8fc.dict | Bin 2189 -> 2264 bytes
.../sandbox-hdp21/kylin.properties | 0
.../test_case_data/sandbox/kylin.properties | 0
script/package.sh | 0
.../kylin/rest/service/ProjectService.java | 2 +-
webapp/app/css/AdminLTE-fonts.css | 0
webapp/app/css/AdminLTE.css | 0
...1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2 | Bin
...t5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2 | Bin
webapp/app/index.html | 2 +
webapp/app/js/controllers/cubeAdvanceSetting.js | 0
webapp/app/js/controllers/cubeDimensions.js | 0
webapp/app/js/controllers/cubeEdit.js | 0
webapp/app/js/controllers/cubeFilter.js | 0
webapp/app/js/controllers/cubeModel.js | 0
webapp/app/js/controllers/cubeRefresh.js | 0
webapp/app/js/controllers/cubeSchema.js | 0
webapp/app/js/controllers/cubes.js | 51 ++++--------
webapp/app/js/controllers/job.js | 41 ++++------
webapp/app/js/controllers/page.js | 0
webapp/app/js/controllers/projects.js | 0
webapp/app/js/controllers/sourceMeta.js | 0
webapp/app/js/filters/filter.js | 0
webapp/app/js/model/cubeConfig.js | 0
webapp/app/js/model/cubeListModel.js | 65 +++++++++++++++
webapp/app/js/model/cubesManager.js | 0
webapp/app/js/model/jobConfig.js | 0
webapp/app/js/model/jobListModel.js | 61 ++++++++++++++
webapp/app/js/model/metaModel.js | 0
webapp/app/js/model/projectConfig.js | 0
webapp/app/js/model/projectModel.js | 0
webapp/app/js/model/tableConfig.js | 0
webapp/app/js/model/tableModel.js | 0
webapp/app/partials/common/access.html | 8 +-
.../cubeDesigner/advanced_settings.html | 3 +
.../app/partials/cubeDesigner/data_model.html | 1 +
.../app/partials/cubeDesigner/dimensions.html | 3 +
.../app/partials/cubeDesigner/incremental.html | 1 +
webapp/app/partials/cubeDesigner/measures.html | 2 +
webapp/app/partials/cubes/cube_json_edit.html | 0
webapp/app/partials/cubes/cubes.html | 14 ++--
webapp/app/partials/jobs/job_submit.html | 1 -
webapp/app/partials/jobs/jobs.html | 14 ++--
webapp/app/partials/projects/projects.html | 0
webapp/app/partials/tables/source_metadata.html | 0
webapp/bower.json | 0
webapp/grunt.json | 0
81 files changed, 191 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/47263143/conf/kylin.properties
----------------------------------------------------------------------
diff --cc conf/kylin.properties
index f607dca,bf86a26..f607dca
mode 100755,100644..100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/47263143/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --cc examples/test_case_data/sandbox-hdp21/kylin.properties
index 3e2f97a,7474283..3e2f97a
mode 100755,100644..100644
--- a/examples/test_case_data/sandbox-hdp21/kylin.properties
+++ b/examples/test_case_data/sandbox-hdp21/kylin.properties
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/47263143/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --cc examples/test_case_data/sandbox/kylin.properties
index cd5d02c,adc0dda..cd5d02c
mode 100755,100644..100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
[19/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/09f4bc3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/09f4bc3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/09f4bc3b
Branch: refs/heads/inverted-index
Commit: 09f4bc3b050832c731b8fe0175fc88e086a409c8
Parents: 5386ccd
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:59:31 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:59:31 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 4 ++--
script/download-tomcat.sh | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09f4bc3b/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
index a6aabdf..735de53 100755
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
@@ -49,11 +49,11 @@ hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.io.tmpdir=${tomcat_root}/temp \
-Dkylin.hive.dependency=${hive_dependency} \
-Dspring.profiles.active=${spring_profile} \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
+org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 &
echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
if [ "$useSandbox" = "true" ]
then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
else
echo "Please visit http://<ip>:7070/kylin"
fi
-echo "You can check the log at ${tomcat_root}/logs/kylin_sandbox.log"
+echo "You can check the log at ${tomcat_root}/logs/kylin.log"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/09f4bc3b/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/script/download-tomcat.sh b/script/download-tomcat.sh
index fdbe440..1751541 100644
--- a/script/download-tomcat.sh
+++ b/script/download-tomcat.sh
@@ -8,13 +8,13 @@ rm -rf tomcat
if [ ! -f "apache-tomcat-7.0.59.tar.gz" ]
then
echo "not binary file found"
- wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
+ wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.59/bin/apache-tomcat-7.0.59.tar.gz || echo "download tomcat failed"
else
if [ ! `md5sum apache-tomcat-7.0.59.tar.gz | awk '{print $1}'` -eq "ec570258976edf9a833cd88fd9220909" ]
then
echo "md5 check failed"
rm apache-tomcat-7.0.59.tar.gz
- wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
+ wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.59/bin/apache-tomcat-7.0.59.tar.gz || echo "download tomcat failed"
fi
fi
[07/50] incubator-kylin git commit: refactor script
Posted by li...@apache.org.
refactor script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5038abed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5038abed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5038abed
Branch: refs/heads/inverted-index
Commit: 5038abed209d1e41cb06bcd72bee780e0c96e4e4
Parents: e590e3e
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 16:49:08 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 16:49:08 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 2 -
conf/kylin.properties | 6 +-
.../test_case_data/localmeta/kylin.properties | 7 +-
.../localmeta_v1/kylin.properties | 7 +-
.../minicluster/b-kylin/kylin.properties | 8 +-
.../minicluster/b-kylin/meta/kylin.properties | 3 -
.../test_case_data/minicluster/kylin.properties | 8 +-
.../sandbox-hdp21/kylin.properties | 7 +-
.../test_case_data/sandbox/kylin.properties | 7 +-
sbin/check-env.sh | 39 -----
sbin/find-hive-dependency.sh | 19 --
sbin/healthmon.sh | 33 ----
sbin/kylin.sh | 35 ----
sbin/sandbox_deploy.sh | 172 -------------------
sbin/start-kylin-sandbox.sh | 50 ------
sbin/start-kylin.sh | 49 ------
sbin/stop-kylin.sh | 4 -
script/build.sh | 17 ++
script/compress.sh | 2 +-
script/debug.sh | 31 ----
script/download-tomcat.sh | 1 +
script/install.sh | 15 --
script/package.sh | 38 ++--
script/prepare.sh | 6 +
script/release.sh | 33 ----
server/ServiceConfig.md | 2 -
26 files changed, 79 insertions(+), 522 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index de8ca57..ef2c6e2 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -59,8 +59,6 @@ public class KylinConfig {
public static final String KYLIN_JOB_CONCURRENT_MAX_LIMIT = "kylin.job.concurrent.max.limit";
-// public static final String KYLIN_JOB_YARN_APP_REST_CHECK_STATUS_URL = "kylin.job.yarn.app.rest.check.status.url";
-
public static final String KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS = "kylin.job.yarn.app.rest.check.interval.seconds";
public static final String KYLIN_TMP_HDFS_DIR = "kylin.tmp.hdfs.dir";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index c292014..bf86a26 100755
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -90,3 +87,6 @@ kylin.web.contact_mail=
#env DEV|QA|PROD
deploy.env=DEV
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 9bbefc5..53d24ee 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -63,3 +60,7 @@ acl.defaultRole=
ganglia.group=
ganglia.port=8664
+###########################config info for sandbox#######################
+kylin.sandbox=true
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/localmeta_v1/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta_v1/kylin.properties b/examples/test_case_data/localmeta_v1/kylin.properties
index 9b8b90a..96439b4 100644
--- a/examples/test_case_data/localmeta_v1/kylin.properties
+++ b/examples/test_case_data/localmeta_v1/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -63,3 +60,7 @@ acl.defaultRole=
ganglia.group=
ganglia.port=8664
+###########################config info for sandbox#######################
+kylin.sandbox=true
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/minicluster/b-kylin/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties
index 699a91b..19c8de6 100644
--- a/examples/test_case_data/minicluster/b-kylin/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=1
@@ -60,4 +57,7 @@ ldap.service.groupSearchBase=
acl.adminRole=
acl.defaultRole=
ganglia.group=
-ganglia.port=8664
\ No newline at end of file
+ganglia.port=8664
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
index 699a91b..419b9e8 100644
--- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=1
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index 53f003c..7816cad 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -45,9 +45,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://127.0.0.1:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -66,4 +63,7 @@ ldap.service.groupSearchBase=
acl.adminRole=
acl.defaultRole=
ganglia.group=
-ganglia.port=8664
\ No newline at end of file
+ganglia.port=8664
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties
index a56ffd1..7474283 100755
--- a/examples/test_case_data/sandbox-hdp21/kylin.properties
+++ b/examples/test_case_data/sandbox-hdp21/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -70,3 +67,7 @@ mail.host=
mail.username=
mail.password=
mail.sender=
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index a79946b..adc0dda 100755
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,9 +39,6 @@ kylin.job.remote.cli.working.dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.concurrent.max.limit=10
-# The url to check hadoop job status
-kylin.job.yarn.app.rest.check.status.url=http://sandbox:8088/ws/v1/cluster/apps/${job_id}?anonymous=true
-
# Time interval to check hadoop job status
kylin.job.yarn.app.rest.check.interval.seconds=10
@@ -90,3 +87,7 @@ kylin.web.contact_mail=
#env DEV|QA|PROD
deploy.env=DEV
+
+###########################config info for sandbox#######################
+kylin.sandbox=true
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/check-env.sh
----------------------------------------------------------------------
diff --git a/sbin/check-env.sh b/sbin/check-env.sh
deleted file mode 100755
index 3bef6c4..0000000
--- a/sbin/check-env.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-
-echo "Checking KYLIN_HOME..."
-if [ -z "$KYLIN_HOME" ]
-then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
-else
- echo "KYLIN_HOME is set to ${KYLIN_HOME}"
-fi
-
-echo "Checking hbase..."
-if [ -z "$(command -v hbase version)" ]
-then
- echo "Please make sure the user has the privilege to run hbase shell"
- exit 1
-else
- echo "hbase check passed"
-fi
-
-echo "Checking hive..."
-if [ -z "$(command -v hive --version)" ]
-then
- echo "Please make sure the user has the privilege to run hive shell"
- exit 1
-else
- echo "hive check passed"
-fi
-
-echo "Checking hadoop..."
-if [ -z "$(command -v hadoop version)" ]
-then
- echo "Please make sure the user has the privilege to run hadoop shell"
- exit 1
-else
- echo "hadoop check passed"
-fi
-
-exit 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/sbin/find-hive-dependency.sh b/sbin/find-hive-dependency.sh
deleted file mode 100644
index e03495a..0000000
--- a/sbin/find-hive-dependency.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-hive_env=`hive -e set | grep 'env:CLASSPATH'`
-
-hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
-arr=(`echo $hive_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
-hive_exec_path=
-for data in ${arr[@]}
-do
- result=`echo $data | grep 'hive-exec.jar'`
- if [ $result ]
- then
- hive_exec_path=$data
- fi
-done
-hdp_home=`echo $hive_exec_path | awk -F '/hive/lib/' '{print $1}'`
-
-hive_dependency=/usr/hdp/current/hive-client/conf/:${hdp_home}/hive/lib/*:${hdp_home}/hive-hcatalog/share/hcatalog/*
-export hive_dependency
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/healthmon.sh
----------------------------------------------------------------------
diff --git a/sbin/healthmon.sh b/sbin/healthmon.sh
deleted file mode 100644
index 439eef4..0000000
--- a/sbin/healthmon.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-ALERT="your@email.com"
-
-OUTPUT=$(
- curl --max-time 20 -# \
- --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
- -H "Authorization:Basic QURNSU46S1lMSU4=" \
- -H "Content-Type:application/json;charset=UTF-8" \
- http://localhost:7070/kylin/api/query \
-)
-
-# ----------------------------------------------------------------------------
-
-date
-
-if [[ $OUTPUT == *"results"* ]]; then
- echo "Good."
-else
- echo "Bad."
- TS_FILE=/tmp/kylin_healthmon_ts
- LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
- CURR_TS=`date +%s`
- echo last: $LAST_TS
- echo curr: $CURR_TS
- if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
- echo "Sending mail..."
- echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
- if [ "$?" == "0" ]; then
- touch $TS_FILE
- fi
- fi
-fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/kylin.sh b/sbin/kylin.sh
deleted file mode 100755
index b264720..0000000
--- a/sbin/kylin.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
-then
- echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
-else
- exit 1
-fi
-
-#The location of all hadoop/hbase configurations are difficult to get.
-#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
-#For example, in hdp 2.2, there is a system property called hdp.version,
-#which we cannot get until running hbase or hadoop shell command.
-#
-#To save all these troubles, we use hbase runjar to start tomcat.
-#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
-#hbase command will do all the dirty tasks for us:
-if [ "$1" = "start" ] ; then
- export HBASE_CLASSPATH_PREFIX=/etc/kylin:${CATALINA_HOME}/bin/bootstrap.jar:${CATALINA_HOME}/bin/tomcat-juli.jar:${CATALINA_HOME}/lib/*:$HBASE_CLASSPATH_PREFIX
- hbase -Djava.util.logging.config.file=${CATALINA_HOME}/conf/logging.properties -Djava.library.path=${KYLIN_LD_LIBRARY_PATH} -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true -Dspring.profiles.active=sandbox -Djava.endorsed.dirs=${CATALINA_HOME}/endorsed -Dcatalina.base=${CATALINA_HOME} -Dcatalina.home=${CATALINA_HOME} -Djava.io.tmpdir=${CATALINA_HOME}/temp org.apache.hadoop.util.RunJar ${CATALINA_HOME}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${CATALINA_HOME}/logs/kylin_sandbox.log 2>&1 &
- echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
- echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
- echo "You can check the log at ${CATALINA_HOME}/logs/kylin_sandbox.log"
-
-elif [ "$1" = "stop" ]; then
- # kill all tomcats started by current user
- ps -fu $USER | grep tomcat | grep -v "grep" | awk '{print $2}' | xargs kill
- echo "all tomcats started by $USER are killed"
-
-else
- echo "Usage: \"kylin.sh start\", or \"kylin.sh stop\""
-
-fi
-
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/sandbox_deploy.sh
----------------------------------------------------------------------
diff --git a/sbin/sandbox_deploy.sh b/sbin/sandbox_deploy.sh
deleted file mode 100755
index 7202f97..0000000
--- a/sbin/sandbox_deploy.sh
+++ /dev/null
@@ -1,172 +0,0 @@
-#!/bin/bash
-
-set -o pipefail # trace ERR through pipes
-set -o errtrace # trace ERR through 'time command' and other functions
-
-function error() {
-SCRIPT="$0" # script name
-LASTLINE="$1" # line of error occurrence
-LASTERR="$2" # error code
-echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code ${LASTERR}"
-exit 1
-}
-
-function escape_sed_replacement(){
- v=$1
- v=$(sed -e 's/[\/&]/\\&/g' <<< $v)
- echo $v
-}
-
-function escape_sed_pattern(){
- v=$1
- v=$(sed -e 's/[]\/$*.^|[]/\\&/g' <<< $v)
- echo $v
-}
-
-trap 'error ${LINENO} ${?}' ERR
-
-echo ""
-echo "Welcome to use Kylin-Deploy script"
-echo "This script will help you:"
-echo "1. Check environment"
-echo "2. Build Kylin artifacts"
-echo "3. Prepare test cube related data"
-echo "Please make sure you are running this script on a hadoop CLI machine, and you have enough permissions."
-echo "Also, We assume you have installed: JAVA, TOMCAT, NPM and MAVEN."
-echo "[Warning] The installation may break existing tomcat applications on this CLI"
-echo ""
-
-
-[[ "$SILENT" ]] || ( read -p "Are you sure you want to proceed?(press Y or y to confirm) " -n 1 -r
-echo # (optional) move to a new line
-if [[ ! $REPLY =~ ^[Yy]$ ]]
-then
- echo "Not going to proceed, quit without finishing! You can rerun the script to have another try."
- exit 1
-fi )
-
-echo "Checking JAVA status..."
-
-if [ -z "$JAVA_HOME" ]
-then
- echo "Please set JAVA_HOME so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "JAVA_HOME is set to $JAVA_HOME"
-fi
-
-if [ -d "$JAVA_HOME" ]
-then
- echo "$JAVA_HOME exists"
-else
- echo " $JAVA_HOME does not exist or is not a directory."
- exit 1
-fi
-
-echo "Checking tomcat status..."
-
-if [ -z "$CATALINA_HOME" ]
-then
- echo "Please set CATALINA_HOME so that Kylin-Deploy knows where to start tomcat"
- exit 1
-else
- echo "CATALINA_HOME is set to $CATALINA_HOME"
-fi
-
-if [ -d "$CATALINA_HOME" ]
-then
- echo "$CATALINA_HOME exists"
-else
- echo " $CATALINA_HOME does not exist or is not a directory."
- exit 1
-fi
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
- echo "Please install maven first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "maven check passed"
-fi
-
-echo "Checking npm..."
-
-if [ -z "$(command -v npm)" ]
-then
- echo "Please install npm first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "npm check passed"
-fi
-
-KYLIN_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo "Kylin home folder path is $KYLIN_HOME"
-cd $KYLIN_HOME
-
-
-echo "Building and packaging..."
-source ./package.sh
-
-echo "retrieving classpath..."
-cd $KYLIN_HOME/job/target
-JOB_JAR_NAME="kylin-job-latest.jar"
-#export hbase configs, most of the configurations are useless now, but KYLIN_HBASE_CONF_PATH is used by SampleCubeSetupTest now
-hbase org.apache.hadoop.util.RunJar $JOB_JAR_NAME org.apache.kylin.job.deployment.HbaseConfigPrinterCLI /tmp/kylin_retrieve.sh
-#load config variables
-source /tmp/kylin_retrieve.sh
-
-cd $KYLIN_HOME
-mkdir -p /etc/kylin
-
-HOSTNAME=`hostname`
-DEFAULT_CHECK_URL="kylin.job.yarn.app.rest.check.status.url=http://sandbox"
-DEFAULT_SERVER_LIST="kylin.rest.servers=localhost"
-NEW_CHECK_URL_PREFIX="kylin.job.yarn.app.rest.check.status.url=http://"
-NEW_SERVER_LIST_PREFIX="kylin.rest.servers="
-
-#escape special characters for sed
-DEFAULT_CHECK_URL=$(escape_sed_pattern $DEFAULT_CHECK_URL)
-DEFAULT_SERVER_LIST=$(escape_sed_pattern $DEFAULT_SERVER_LIST)
-NEW_CHECK_URL_PREFIX=$(escape_sed_replacement $NEW_CHECK_URL_PREFIX)
-NEW_SERVER_LIST_PREFIX=$(escape_sed_replacement $NEW_SERVER_LIST_PREFIX)
-HOSTNAME=$(escape_sed_replacement $HOSTNAME)
-
-#deploy kylin.properties to /etc/kylin
-cat examples/test_case_data/sandbox/kylin.properties | \
- sed -e "s/${DEFAULT_CHECK_URL}/${NEW_CHECK_URL_PREFIX}${HOSTNAME}/g" | \
- sed -e "s/${DEFAULT_SERVER_LIST}/${NEW_SERVER_LIST_PREFIX}${HOSTNAME}/g" > /etc/kylin/kylin.properties
-
-
-
-# 1. generate synthetic fact table(test_kylin_fact) data and dump it into hive
-# 2. create empty cubes on these data, ready to be built
-cd $KYLIN_HOME
-mvn test -Dtest=org.apache.kylin.job.SampleCubeSetupTest -DfailIfNoTests=false
-
-#overwrite server.xml
-mv ${CATALINA_HOME}/conf/server.xml ${CATALINA_HOME}/conf/server.xml.bak
-cp deploy/server.xml ${CATALINA_HOME}/conf/server.xml
-echo "server.xml overwritten..."
-
-#deploy kylin.war
-rm -rf $CATALINA_HOME/webapps/kylin
-rm -f $CATALINA_HOME/webapps/kylin.war
-cp $KYLIN_HOME/server/target/kylin.war $CATALINA_HOME/webapps/
-chmod 644 $CATALINA_HOME/webapps/kylin.war
-echo "Tomcat war deployed..."
-
-echo "Kylin is deployed successfully!!!"
-echo ""
-echo ""
-echo "Please check the configuration:"
-echo ""
-echo "==================================================================="
-cat /etc/kylin/kylin.properties
-echo "==================================================================="
-echo ""
-echo "You can directly modify it by editing /etc/kylin/kylin.properties"
-echo "If you're using standard hadoop sandbox, you might keep it untouched"
-echo "After checking, please start kylin by using \"./kylin.sh start\""
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/start-kylin-sandbox.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin-sandbox.sh b/sbin/start-kylin-sandbox.sh
deleted file mode 100644
index b8912f2..0000000
--- a/sbin/start-kylin-sandbox.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/sh
-
-dir=$(dirname ${0})
-
-tomcat_root=${dir}/../tomcat
-export tomcat_root
-
-sh ${dir}/check-env.sh || { exit 1; }
-
-#if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
-#then
-# echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
-#else
-# exit 1
-#fi
-
-#The location of all hadoop/hbase configurations are difficult to get.
-#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
-#For example, in hdp 2.2, there is a system property called hdp.version,
-#which we cannot get until running hbase or hadoop shell command.
-#
-#To save all these troubles, we use hbase runjar to start tomcat.
-#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
-#hbase command will do all the dirty tasks for us:
-
-#-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
-
-
-rm -rf ${tomcat_root}/webapps/*
-cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
-
-source ${dir}/find-hive-dependency.sh
-
-export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
-export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-
-hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
--Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
--Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
--Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
--Djava.endorsed.dirs=${tomcat_root}/endorsed \
--Dcatalina.base=${tomcat_root} \
--Dcatalina.home=${tomcat_root} \
--Djava.io.tmpdir=${tomcat_root}/temp \
--Dkylin.hive.dependency=${hive_dependency} \
--Dspring.profiles.active=sandbox \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
-echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
-echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
-echo "You can check the log at ${tomcat_root}/logs/kylin_sandbox.log"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin.sh b/sbin/start-kylin.sh
deleted file mode 100644
index d7436f4..0000000
--- a/sbin/start-kylin.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/sh
-
-dir=$(dirname ${0})
-
-tomcat_root=${dir}/../tomcat
-export tomcat_root
-
-sh ${dir}/check-env.sh || { exit 1; }
-
-#if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
-#then
-# echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
-#else
-# exit 1
-#fi
-
-#The location of all hadoop/hbase configurations are difficult to get.
-#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
-#For example, in hdp 2.2, there is a system property called hdp.version,
-#which we cannot get until running hbase or hadoop shell command.
-#
-#To save all these troubles, we use hbase runjar to start tomcat.
-#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
-#hbase command will do all the dirty tasks for us:
-
-#-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
-
-
-rm -rf ${tomcat_root}/webapps/*
-cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
-
-source ${dir}/find-hive-dependency.sh
-
-export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
-export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-
-hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
--Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
--Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
--Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
--Djava.endorsed.dirs=${tomcat_root}/endorsed \
--Dcatalina.base=${tomcat_root} \
--Dcatalina.home=${tomcat_root} \
--Djava.io.tmpdir=${tomcat_root}/temp \
--Dkylin.hive.dependency=${hive_dependency} \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
-echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
-echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
-echo "You can check the log at ${tomcat_root}/logs/kylin_sandbox.log"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/sbin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/stop-kylin.sh b/sbin/stop-kylin.sh
deleted file mode 100644
index 806513b..0000000
--- a/sbin/stop-kylin.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-ps -fu $USER | grep tomcat | grep -v "grep" | awk '{print $2}' | xargs kill
-echo "all tomcats started by $USER are killed"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/build.sh
----------------------------------------------------------------------
diff --git a/script/build.sh b/script/build.sh
new file mode 100644
index 0000000..08b2b0a
--- /dev/null
+++ b/script/build.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+echo "package back-end"
+
+dir=$(dirname ${0})
+cd ${dir}/..
+
+mvn clean install -DskipTests
+
+#package webapp
+echo 'package front-end'
+cd webapp
+npm install -g bower
+bower install
+npm install
+npm install -g grunt-cli
+grunt dev --buildEnv=dev
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index ad0b352..6221b2b 100644
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -13,6 +13,6 @@ fi
#package tgz
echo 'package tgz'
-tar -cvzf kylin-${version}.tgz lib sbin conf tomcat
+tar -cvzf kylin-${version}.tgz lib bin conf tomcat
rm -rf tomcat/webapps/*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/debug.sh
----------------------------------------------------------------------
diff --git a/script/debug.sh b/script/debug.sh
deleted file mode 100644
index bbf575f..0000000
--- a/script/debug.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
- echo "Please install maven first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "maven check passed"
-fi
-
-echo "Checking npm..."
-
-if [ -z "$(command -v npm)" ]
-then
- echo "Please install npm first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "npm check passed"
-fi
-
-dir=$(dirname ${0})
-cd ${dir}/..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
-echo "kylin version: ${version}"
-export version
-
-sh script/package.sh || { exit 1; }
-sh script/prepare.sh || { exit 1; }
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/script/download-tomcat.sh b/script/download-tomcat.sh
index 24ec279..a01b132 100644
--- a/script/download-tomcat.sh
+++ b/script/download-tomcat.sh
@@ -9,6 +9,7 @@ wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7
tar -zxvf apache-tomcat-7.0.57.tar.gz
mv apache-tomcat-7.0.57 tomcat
+rm -rf tomcat/webapps/*
mv tomcat/conf/server.xml tomcat/conf/server.xml.bak
cp deploy/server.xml tomcat/conf/server.xml
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/install.sh
----------------------------------------------------------------------
diff --git a/script/install.sh b/script/install.sh
deleted file mode 100755
index 6792fe0..0000000
--- a/script/install.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-
-cd ~
-wget http://apache.cs.uu.nl/dist/tomcat/tomcat-7/v7.0.56/bin/apache-tomcat-7.0.56.tar.gz
-tar -xzvf apache-tomcat-7.0.56.tar.gz
-export CATALINA_HOME=/root/apache-tomcat-7.0.56
-
-wget http://apache.proserve.nl/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
-tar -xzvf apache-maven-3.2.3-bin.tar.gz
-ln -s /root/apache-maven-3.2.3/bin/mvn /usr/bin/mvn
-
-wget http://nodejs.org/dist/v0.10.32/node-v0.10.32-linux-x64.tar.gz
-tar -xzvf node-v0.10.32-linux-x64.tar.gz
-export PATH=/root/node-v0.10.32-linux-x64/bin:$PATH
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
old mode 100644
new mode 100755
index 08b2b0a..6252690
--- a/script/package.sh
+++ b/script/package.sh
@@ -1,17 +1,33 @@
-#!/bin/sh
+#!/bin/bash
-echo "package back-end"
+echo "Checking maven..."
+
+if [ -z "$(command -v mvn)" ]
+then
+ echo "Please install maven first so that Kylin-Deploy can proceed"
+ exit 1
+else
+ echo "maven check passed"
+fi
+
+echo "Checking npm..."
+
+if [ -z "$(command -v npm)" ]
+then
+ echo "Please install npm first so that Kylin-Deploy can proceed"
+ exit 1
+else
+ echo "npm check passed"
+fi
dir=$(dirname ${0})
cd ${dir}/..
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+echo "kylin version: ${version}"
+export version
-mvn clean install -DskipTests
+sh script/build.sh || { exit 1; }
+sh script/prepare.sh || { exit 1; }
+sh script/download-tomcat.sh || { exit 1; }
+sh scirpt/compress.sh || { exit 1; }
-#package webapp
-echo 'package front-end'
-cd webapp
-npm install -g bower
-bower install
-npm install
-npm install -g grunt-cli
-grunt dev --buildEnv=dev
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/prepare.sh
----------------------------------------------------------------------
diff --git a/script/prepare.sh b/script/prepare.sh
index fe615fc..8a855e1 100644
--- a/script/prepare.sh
+++ b/script/prepare.sh
@@ -18,6 +18,12 @@ cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
echo "add js css to war"
+if [ ! -d "webapp/dist" ]
+then
+ echo "error generate js files"
+ exit 1
+fi
+
cd webapp/dist
for f in * .[^.]*
do
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/script/release.sh
----------------------------------------------------------------------
diff --git a/script/release.sh b/script/release.sh
deleted file mode 100755
index e0de140..0000000
--- a/script/release.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-echo "Checking maven..."
-
-if [ -z "$(command -v mvn)" ]
-then
- echo "Please install maven first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "maven check passed"
-fi
-
-echo "Checking npm..."
-
-if [ -z "$(command -v npm)" ]
-then
- echo "Please install npm first so that Kylin-Deploy can proceed"
- exit 1
-else
- echo "npm check passed"
-fi
-
-dir=$(dirname ${0})
-cd ${dir}/..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
-echo "kylin version: ${version}"
-export version
-
-sh script/package.sh || { exit 1; }
-sh script/prepare.sh || { exit 1; }
-sh script/download-tomcat.sh || { exit 1; }
-sh scirpt/compress.sh || { exit 1; }
-
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5038abed/server/ServiceConfig.md
----------------------------------------------------------------------
diff --git a/server/ServiceConfig.md b/server/ServiceConfig.md
index 723557e..0f951be 100644
--- a/server/ServiceConfig.md
+++ b/server/ServiceConfig.md
@@ -33,10 +33,8 @@
##### kylin.job.remote.cli.password
##### kylin.job.remote.cli.working.dir
##### kylin.job.concurrent.max.limit
-##### kylin.job.yarn.app.rest.check.status.url
##### kylin.job.command.exe.type
##### kylin.job.admin.dls
##### kylin.job.step.timeout
-##### kylin.job.yarn.app.rest.check.status.url
##### kylin.job.yarn.app.rest.check.interval.seconds
##### kylin.job.concurrent.max.limit
\ No newline at end of file
[34/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5edcf515
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5edcf515
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5edcf515
Branch: refs/heads/inverted-index
Commit: 5edcf515b6bdb2af0730918603140422cc20e068
Parents: 8d84c1f 443b192
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:50:03 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:50:03 2015 +0800
----------------------------------------------------------------------
bin/stop-kylin.sh | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[36/50] incubator-kylin git commit: Use Integer.MAX_VALUE as the limit on getting project list if param “limit” is not presented.
Posted by li...@apache.org.
Use Integer.MAX_VALUE as the limit on getting project list if param “limit” is not presented.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7fefad5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7fefad5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7fefad5f
Branch: refs/heads/inverted-index
Commit: 7fefad5f1991f5476325b2cf1bc3ae678f89513a
Parents: 69959be
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Feb 11 16:03:42 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Feb 11 16:03:42 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/ProjectService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7fefad5f/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index bb91652..9b6464c 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -87,7 +87,7 @@ public class ProjectService extends BasicService {
public List<ProjectInstance> listAllProjects(final Integer limit, final Integer offset) {
List<ProjectInstance> projects = getProjectManager().listAllProjects();
- int climit = (null == limit) ? 30 : limit;
+ int climit = (null == limit) ? Integer.MAX_VALUE : limit;
int coffset = (null == offset) ? 0 : offset;
if (projects.size() <= coffset) {
[26/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c714097e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c714097e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c714097e
Branch: refs/heads/inverted-index
Commit: c714097eedf7110b6b326c9e820191be33c857d5
Parents: 75431e3
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 15:39:15 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 15:39:15 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 2 +-
bin/stop-kylin.sh | 12 ++++++++++--
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c714097e/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
index 9225fb6..b2544a5 100755
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
@@ -46,7 +46,7 @@ hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.io.tmpdir=${tomcat_root}/temp \
-Dkylin.hive.dependency=${hive_dependency} \
-Dspring.profiles.active=${spring_profile} \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 &
+org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
if [ "$useSandbox" = "true" ]
then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c714097e/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
index 806513b..4ac8b7c 100755
--- a/bin/stop-kylin.sh
+++ b/bin/stop-kylin.sh
@@ -1,4 +1,12 @@
#!/bin/sh
-ps -fu $USER | grep tomcat | grep -v "grep" | awk '{print $2}' | xargs kill
-echo "all tomcats started by $USER are killed"
\ No newline at end of file
+pid=`cat ${KYLIN_HOME}/pid || echo "no pid found";exit 1`
+if [ ! "$pid" = "" ]
+then
+ echo "no pid found"
+ exit 1
+else
+ echo "stopping pid:$pid"
+ kill $pid
+fi
+rm ${KYLIN_HOME}/pid
[49/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ed729eef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ed729eef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ed729eef
Branch: refs/heads/inverted-index
Commit: ed729eefd4ad63de40af350ca4ff116e9445a33b
Parents: be5e23f 1ce77a5
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Feb 12 13:20:33 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Feb 12 13:20:33 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 567 ++++++++++---------
.../org/apache/kylin/common/util/BasicTest.java | 23 +-
.../common/util/HBaseMiniclusterHelper.java | 2 -
.../common/util/HyperLogLogCounterTest.java | 12 +
docs/website/_includes/footer.html | 14 +-
docs/website/_includes/head.cn.html | 42 ++
docs/website/_includes/header.cn.html | 39 ++
docs/website/_includes/header.html | 21 +-
docs/website/_layouts/default-cn.html | 23 +
docs/website/_layouts/post.html | 11 +-
.../_posts/2015-01-25-introduce-data-model.md | 40 ++
docs/website/about/index.md | 77 +--
docs/website/assets/css/styles.css | 32 +-
docs/website/blog/index.html | 21 +-
docs/website/community/index.html | 8 +-
docs/website/css/main.scss | 52 ++
docs/website/docs/index.html | 2 -
docs/website/index.cn.md | 91 +++
docs/website/index.md | 91 +++
.../invertedindex/index/RawTableRecord.java | 20 +-
.../apache/kylin/invertedindex/index/Slice.java | 330 ++++++-----
.../kylin/invertedindex/index/TableRecord.java | 12 +-
.../invertedindex/index/TableRecordInfo.java | 3 +-
.../kylin/metadata/measure/HLLCSerializer.java | 2 +-
.../measure/fixedlen/FixedHLLCodec.java | 58 ++
.../measure/fixedlen/FixedLenMeasureCodec.java | 4 +-
.../measure/fixedlen/FixedPointLongCodec.java | 6 +-
.../endpoint/EndpointAggregators.java | 161 ++++--
.../endpoint/EndpointTupleIterator.java | 18 +-
.../hbase/coprocessor/endpoint/IIEndpoint.java | 2 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 12 +-
31 files changed, 1148 insertions(+), 648 deletions(-)
----------------------------------------------------------------------
[12/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c94519ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c94519ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c94519ac
Branch: refs/heads/inverted-index
Commit: c94519ac3f7937fa9e4b525e08c9a09414c94f58
Parents: 20e58b0
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:26:48 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:26:48 2015 +0800
----------------------------------------------------------------------
script/download-tomcat.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c94519ac/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/script/download-tomcat.sh b/script/download-tomcat.sh
index 3afc0df..fdbe440 100644
--- a/script/download-tomcat.sh
+++ b/script/download-tomcat.sh
@@ -3,15 +3,16 @@
dir=$(dirname ${0})
cd ${dir}/..
-rm apache-tomcat-7.0.57.tar.gz
rm -rf tomcat
if [ ! -f "apache-tomcat-7.0.59.tar.gz" ]
then
+ echo "not binary file found"
wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
else
if [ ! `md5sum apache-tomcat-7.0.59.tar.gz | awk '{print $1}'` -eq "ec570258976edf9a833cd88fd9220909" ]
then
+ echo "md5 check failed"
rm apache-tomcat-7.0.59.tar.gz
wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
fi
[25/50] incubator-kylin git commit: remove antlr3 plugin
Posted by li...@apache.org.
remove antlr3 plugin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/75431e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/75431e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/75431e34
Branch: refs/heads/inverted-index
Commit: 75431e344112f3323371773588ede556e6f0c54d
Parents: 5712b1d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 14:15:24 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 14:15:24 2015 +0800
----------------------------------------------------------------------
pom.xml | 15 ---------------
1 file changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/75431e34/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d7b945d..0a85ba2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -503,21 +503,6 @@
</configuration>
</plugin>
<plugin>
- <groupId>org.antlr</groupId>
- <artifactId>antlr3-maven-plugin</artifactId>
- <version>${antlr.version}</version>
- <executions>
- <execution>
- <configuration>
- <goals>
- <goal>antlr</goal>
- </goals>
- <debug>false</debug>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.6</version>
[28/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f460a14a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f460a14a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f460a14a
Branch: refs/heads/inverted-index
Commit: f460a14a82a3240e332dde61f42cafae06ed9e53
Parents: 40ff1cd c714097
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Wed Feb 11 15:43:48 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Wed Feb 11 15:43:48 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 5 +----
bin/stop-kylin.sh | 12 ++++++++++--
pom.xml | 15 ---------------
script/compress.sh | 8 +++++---
script/package.sh | 2 +-
script/prepare.sh | 4 ++--
6 files changed, 19 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
[11/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/20e58b0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/20e58b0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/20e58b0e
Branch: refs/heads/inverted-index
Commit: 20e58b0eb22219dcf6e9122c717de9b14eef4c78
Parents: 79765ba
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:20:45 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:20:45 2015 +0800
----------------------------------------------------------------------
script/download-tomcat.sh | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/20e58b0e/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/script/download-tomcat.sh b/script/download-tomcat.sh
index a01b132..3afc0df 100644
--- a/script/download-tomcat.sh
+++ b/script/download-tomcat.sh
@@ -5,10 +5,20 @@ cd ${dir}/..
rm apache-tomcat-7.0.57.tar.gz
rm -rf tomcat
-wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.57.tar.gz
-tar -zxvf apache-tomcat-7.0.57.tar.gz
-mv apache-tomcat-7.0.57 tomcat
+if [ ! -f "apache-tomcat-7.0.59.tar.gz" ]
+then
+ wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
+else
+ if [ ! `md5sum apache-tomcat-7.0.59.tar.gz | awk '{print $1}'` -eq "ec570258976edf9a833cd88fd9220909" ]
+ then
+ rm apache-tomcat-7.0.59.tar.gz
+ wget http://mirror.sdunix.com/apache/tomcat/tomcat-7/v7.0.57/bin/apache-tomcat-7.0.59.tar.gz
+ fi
+fi
+
+tar -zxvf apache-tomcat-7.0.59.tar.gz
+mv apache-tomcat-7.0.59 tomcat
rm -rf tomcat/webapps/*
mv tomcat/conf/server.xml tomcat/conf/server.xml.bak
[16/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/91545438
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/91545438
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/91545438
Branch: refs/heads/inverted-index
Commit: 91545438a8c86ecb2ad57487ce85c25852149f0a
Parents: a7b4ac8 681314c
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Feb 10 17:39:21 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Feb 10 17:39:21 2015 +0800
----------------------------------------------------------------------
.gitignore | 2 +-
bin/check-env.sh | 39 +++++
bin/find-hive-dependency.sh | 20 +++
bin/healthmon.sh | 33 ++++
bin/start-kylin.sh | 59 +++++++
bin/stop-kylin.sh | 4 +
.../org/apache/kylin/common/KylinConfig.java | 2 -
conf/kylin.properties | 6 +-
.../test_case_data/localmeta/kylin.properties | 7 +-
.../localmeta_v1/kylin.properties | 7 +-
.../minicluster/b-kylin/kylin.properties | 8 +-
.../minicluster/b-kylin/meta/kylin.properties | 3 -
.../test_case_data/minicluster/kylin.properties | 8 +-
.../sandbox-hdp21/kylin.properties | 7 +-
.../sandbox-hdp21/mapred-site.xml | 2 +-
.../test_case_data/sandbox/kylin.properties | 7 +-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
sbin/check-env.sh | 39 -----
sbin/find-hive-dependency.sh | 19 --
sbin/healthmon.sh | 33 ----
sbin/kylin.sh | 35 ----
sbin/sandbox_deploy.sh | 172 -------------------
sbin/start-kylin-sandbox.sh | 50 ------
sbin/start-kylin.sh | 49 ------
sbin/stop-kylin.sh | 4 -
script/build.sh | 17 ++
script/compress.sh | 2 +-
script/debug.sh | 31 ----
script/download-tomcat.sh | 20 ++-
script/install.sh | 15 --
script/package.sh | 38 ++--
script/prepare.sh | 6 +
script/release.sh | 33 ----
server/ServiceConfig.md | 2 -
webapp/CHANGELOG.md | 1 -
webapp/app/js/controllers/page.js | 1 -
webapp/app/js/controllers/projects.js | 3 +-
webapp/app/js/controllers/sourceMeta.js | 13 +-
webapp/app/js/model/projectModel.js | 2 +
webapp/app/js/model/tableModel.js | 7 +-
webapp/app/partials/cubes/cube_json_edit.html | 2 +-
webapp/app/partials/jobs/job_submit.html | 38 ++--
webapp/bower.json | 5 +-
43 files changed, 289 insertions(+), 564 deletions(-)
----------------------------------------------------------------------
[18/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5386ccd6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5386ccd6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5386ccd6
Branch: refs/heads/inverted-index
Commit: 5386ccd6912494ac185d60d30c29291272e9f581
Parents: 376941b 9154543
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:49:58 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:49:58 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CacheController.java | 13 ++-
.../kylin/rest/controller/CubeController.java | 115 +++++++++----------
.../kylin/rest/controller/ModelController.java | 6 +-
.../kylin/rest/controller/TableController.java | 46 ++++----
.../apache/kylin/rest/service/CubeService.java | 79 ++++++-------
.../rest/controller/CacheControllerTest.java | 57 +++++++++
.../rest/controller/CubeControllerTest.java | 47 ++++++--
.../rest/controller/TableControllerTest.java | 74 ++++++++++++
.../kylin/rest/service/ServiceTestBase.java | 22 ++--
9 files changed, 305 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
[06/50] incubator-kylin git commit: config pergem size
Posted by li...@apache.org.
config pergem size
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/9fb8aaab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/9fb8aaab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/9fb8aaab
Branch: refs/heads/inverted-index
Commit: 9fb8aaabfd613bd30e07630271bb201c1debbb2a
Parents: 1553530
Author: honma <ho...@ebay.com>
Authored: Tue Feb 10 16:14:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Feb 10 16:14:50 2015 +0800
----------------------------------------------------------------------
sbin/kylin.sh | 2 +-
sbin/start-kylin-sandbox.sh | 2 +-
sbin/start-kylin.sh | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9fb8aaab/sbin/kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/kylin.sh b/sbin/kylin.sh
index b264720..2bc0764 100755
--- a/sbin/kylin.sh
+++ b/sbin/kylin.sh
@@ -17,7 +17,7 @@ fi
#hbase command will do all the dirty tasks for us:
if [ "$1" = "start" ] ; then
export HBASE_CLASSPATH_PREFIX=/etc/kylin:${CATALINA_HOME}/bin/bootstrap.jar:${CATALINA_HOME}/bin/tomcat-juli.jar:${CATALINA_HOME}/lib/*:$HBASE_CLASSPATH_PREFIX
- hbase -Djava.util.logging.config.file=${CATALINA_HOME}/conf/logging.properties -Djava.library.path=${KYLIN_LD_LIBRARY_PATH} -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true -Dspring.profiles.active=sandbox -Djava.endorsed.dirs=${CATALINA_HOME}/endorsed -Dcatalina.base=${CATALINA_HOME} -Dcatalina.home=${CATALINA_HOME} -Djava.io.tmpdir=${CATALINA_HOME}/temp org.apache.hadoop.util.RunJar ${CATALINA_HOME}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${CATALINA_HOME}/logs/kylin_sandbox.log 2>&1 &
+ hbase -XX:PermSize=256M -XX:MaxPermSize=512M -Djava.util.logging.config.file=${CATALINA_HOME}/conf/logging.properties -Djava.library.path=${KYLIN_LD_LIBRARY_PATH} -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true -Dspring.profiles.active=sandbox -Djava.endorsed.dirs=${CATALINA_HOME}/endorsed -Dcatalina.base=${CATALINA_HOME} -Dcatalina.home=${CATALINA_HOME} -Djava.io.tmpdir=${CATALINA_HOME}/temp org.apache.hadoop.util.RunJar ${CATALINA_HOME}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${CATALINA_HOME}/logs/kylin_sandbox.log 2>&1 &
echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
echo "You can check the log at ${CATALINA_HOME}/logs/kylin_sandbox.log"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9fb8aaab/sbin/start-kylin-sandbox.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin-sandbox.sh b/sbin/start-kylin-sandbox.sh
index b8912f2..e31f77a 100644
--- a/sbin/start-kylin-sandbox.sh
+++ b/sbin/start-kylin-sandbox.sh
@@ -34,7 +34,7 @@ source ${dir}/find-hive-dependency.sh
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+hbase -XX:PermSize=256M -XX:MaxPermSize=512M -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/9fb8aaab/sbin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/sbin/start-kylin.sh b/sbin/start-kylin.sh
index d7436f4..5eb9ebf 100644
--- a/sbin/start-kylin.sh
+++ b/sbin/start-kylin.sh
@@ -34,7 +34,7 @@ source ${dir}/find-hive-dependency.sh
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+hbase -XX:PermSize=256M -XX:MaxPermSize=512M -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
@@ -43,7 +43,7 @@ hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Dcatalina.home=${tomcat_root} \
-Djava.io.tmpdir=${tomcat_root}/temp \
-Dkylin.hive.dependency=${hive_dependency} \
-org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
+org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrapstart > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
echo "You can check the log at ${tomcat_root}/logs/kylin_sandbox.log"
\ No newline at end of file
[10/50] incubator-kylin git commit: remove webapp/CHANGELOG.md from
repository
Posted by li...@apache.org.
remove webapp/CHANGELOG.md from repository
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/79765ba6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/79765ba6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/79765ba6
Branch: refs/heads/inverted-index
Commit: 79765ba661d50bca6f7c7b95c57b258e4834fa7e
Parents: 40af1c8
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:01:44 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:01:44 2015 +0800
----------------------------------------------------------------------
.gitignore | 1 +
webapp/CHANGELOG.md | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/79765ba6/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 4fe5496..4d4bf12 100755
--- a/.gitignore
+++ b/.gitignore
@@ -75,6 +75,7 @@ server/.settings/org.eclipse.wst.jsdt.ui.superType.name
webapp/app/WEB-INF/*
webapp/dist
webapp/node_modules:
+webapp/CHANGELOG.md
# package file
tomcat
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/79765ba6/webapp/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/webapp/CHANGELOG.md b/webapp/CHANGELOG.md
deleted file mode 100644
index f8a032f..0000000
--- a/webapp/CHANGELOG.md
+++ /dev/null
@@ -1 +0,0 @@
-Initial release
[44/50] incubator-kylin git commit: Merge branch
'origin/inverted-index'
Posted by li...@apache.org.
Merge branch 'origin/inverted-index'
Conflicts:
common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bde1e31e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bde1e31e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bde1e31e
Branch: refs/heads/inverted-index
Commit: bde1e31e0a31e8f8955a6c37e769744a50766dcc
Parents: c5d329f 4726314
Author: honma <ho...@ebay.com>
Authored: Thu Feb 12 10:59:59 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Feb 12 10:59:59 2015 +0800
----------------------------------------------------------------------
.gitignore | 2 +-
bin/check-env.sh | 39 +
bin/find-hive-dependency.sh | 20 +
bin/health-check.sh | 33 +
bin/start-kylin.sh | 56 ++
bin/stop-kylin.sh | 18 +
.../org/apache/kylin/common/KylinConfig.java | 5 +-
.../common/hll/HyperLogLogPlusCounter.java | 737 +++++++++----------
conf/kylin.properties | 14 +-
docs/Installation/install_and_run.md | 69 ++
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
.../16d8185c-ee6b-4f8c-a919-756d9809f937.dict | Bin 2515 -> 2630 bytes
.../c12ae49d-9dbe-4a58-b169-19afac317696.dict | Bin 2515 -> 2630 bytes
.../eaed91b0-4182-4ee5-a733-1047a622ee29.dict | Bin 2515 -> 2630 bytes
.../ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict | Bin 2515 -> 2630 bytes
.../7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict | Bin 2940 -> 3105 bytes
.../8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict | Bin 2940 -> 3105 bytes
.../ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict | Bin 2940 -> 3105 bytes
.../eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict | Bin 2940 -> 3105 bytes
.../2602386c-debb-4968-8d2f-b52b8215e385.dict | Bin 1841 -> 2000 bytes
.../4243889f-bc81-4807-a975-7041bbbf35e7.dict | Bin 1841 -> 2000 bytes
.../494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict | Bin 1841 -> 2000 bytes
.../8b4b1c06-fb74-486b-a2ad-74420afebcda.dict | Bin 1841 -> 2000 bytes
.../ac520edd-f9d2-419f-a7de-587bfb97dc81.dict | Bin 1841 -> 2000 bytes
.../0410d2c4-4686-40bc-ba14-170042a2de94.dict | Bin 1498 -> 1537 bytes
.../51ff5e6c-22b3-444e-9915-d376a10f20cb.dict | Bin 1498 -> 1537 bytes
.../a4e57e55-48fc-4f25-a9c8-485deed25925.dict | Bin 1498 -> 1537 bytes
.../aceae914-4246-4251-a0c2-692fe7a300df.dict | Bin 1498 -> 1537 bytes
.../b298089f-9656-4693-b9b2-8fea46f06dd5.dict | Bin 1498 -> 1537 bytes
.../c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict | Bin 1498 -> 1537 bytes
.../0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict | Bin 661 -> 640 bytes
.../14fe66b3-5956-498c-bd93-40182cac5510.dict | Bin 661 -> 640 bytes
.../1d383a36-81b9-4177-a822-04eab3683e5b.dict | Bin 661 -> 640 bytes
.../c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict | Bin 661 -> 640 bytes
.../0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict | Bin 2189 -> 2264 bytes
.../31edf35b-ffca-4f24-8229-f87dc34e3087.dict | Bin 2189 -> 2264 bytes
.../652bd393-678a-4f16-a504-fd8ce1229355.dict | Bin 2189 -> 2264 bytes
.../792eb972-d046-48e6-9428-f6a3aed92fad.dict | Bin 2189 -> 2264 bytes
.../914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict | Bin 2189 -> 2264 bytes
.../d25e554e-deac-4e4a-9289-96f0d121d8fc.dict | Bin 2189 -> 2264 bytes
.../test_case_data/localmeta/kylin.properties | 13 +-
.../localmeta_v1/kylin.properties | 13 +-
.../minicluster/b-kylin/kylin.properties | 14 +-
.../minicluster/b-kylin/meta/kylin.properties | 9 -
.../test_case_data/minicluster/kylin.properties | 14 +-
.../sandbox-hdp21/kylin.properties | 13 +-
.../sandbox-hdp21/mapred-site.xml | 2 +-
.../test_case_data/sandbox/kylin.properties | 13 +-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
jdbc/pom.xml | 5 -
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
pom.xml | 15 -
sbin/check-env.sh | 39 -
sbin/find-hive-dependency.sh | 19 -
sbin/healthmon.sh | 33 -
sbin/kylin.sh | 35 -
sbin/sandbox_deploy.sh | 172 -----
sbin/start-kylin-sandbox.sh | 50 --
sbin/start-kylin.sh | 49 --
sbin/stop-kylin.sh | 4 -
script/build.sh | 17 +
script/compress.sh | 8 +-
script/debug.sh | 31 -
script/download-tomcat.sh | 20 +-
script/install.sh | 15 -
script/package.sh | 38 +-
script/prepare.sh | 10 +-
script/release.sh | 33 -
server/ServiceConfig.md | 6 +-
.../kylin/rest/controller/CacheController.java | 13 +-
.../kylin/rest/controller/CubeController.java | 115 ++-
.../kylin/rest/controller/ModelController.java | 6 +-
.../kylin/rest/controller/TableController.java | 46 +-
.../apache/kylin/rest/service/CubeService.java | 79 +-
.../kylin/rest/service/ProjectService.java | 2 +-
.../rest/controller/CacheControllerTest.java | 57 ++
.../rest/controller/CubeControllerTest.java | 47 +-
.../rest/controller/TableControllerTest.java | 74 ++
.../kylin/rest/service/ServiceTestBase.java | 22 +-
webapp/CHANGELOG.md | 1 -
webapp/app/css/AdminLTE-fonts.css | 0
webapp/app/css/AdminLTE.css | 0
...1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2 | Bin
...t5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2 | Bin
webapp/app/index.html | 2 +
webapp/app/js/controllers/cubeAdvanceSetting.js | 0
webapp/app/js/controllers/cubeDimensions.js | 0
webapp/app/js/controllers/cubeEdit.js | 0
webapp/app/js/controllers/cubeFilter.js | 0
webapp/app/js/controllers/cubeModel.js | 0
webapp/app/js/controllers/cubeRefresh.js | 0
webapp/app/js/controllers/cubeSchema.js | 0
webapp/app/js/controllers/cubes.js | 51 +-
webapp/app/js/controllers/job.js | 41 +-
webapp/app/js/controllers/page.js | 0
webapp/app/js/controllers/projects.js | 0
webapp/app/js/controllers/sourceMeta.js | 0
webapp/app/js/filters/filter.js | 0
webapp/app/js/model/cubeConfig.js | 0
webapp/app/js/model/cubeListModel.js | 65 ++
webapp/app/js/model/cubesManager.js | 0
webapp/app/js/model/jobConfig.js | 0
webapp/app/js/model/jobListModel.js | 61 ++
webapp/app/js/model/metaModel.js | 0
webapp/app/js/model/projectConfig.js | 0
webapp/app/js/model/projectModel.js | 0
webapp/app/js/model/tableConfig.js | 0
webapp/app/js/model/tableModel.js | 0
webapp/app/partials/common/access.html | 8 +-
.../cubeDesigner/advanced_settings.html | 3 +
.../app/partials/cubeDesigner/data_model.html | 1 +
.../app/partials/cubeDesigner/dimensions.html | 3 +
.../app/partials/cubeDesigner/incremental.html | 1 +
webapp/app/partials/cubeDesigner/measures.html | 2 +
webapp/app/partials/cubes/cube_json_edit.html | 0
webapp/app/partials/cubes/cubes.html | 14 +-
webapp/app/partials/jobs/job_submit.html | 1 -
webapp/app/partials/jobs/jobs.html | 14 +-
webapp/app/partials/projects/projects.html | 0
webapp/app/partials/tables/source_metadata.html | 0
webapp/bower.json | 0
webapp/grunt.json | 0
123 files changed, 1190 insertions(+), 1226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bde1e31e/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --cc common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 49a6756,467dfde..6ab0611
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@@ -1,382 -1,276 +1,355 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License.
--*/
--
--package org.apache.kylin.common.hll;
--
- import java.io.ByteArrayInputStream;
- import java.io.ByteArrayOutputStream;
--import java.io.IOException;
--import java.nio.ByteBuffer;
--import java.nio.charset.Charset;
--import java.util.Arrays;
- import java.util.zip.GZIPInputStream;
- import java.util.zip.GZIPOutputStream;
--
- import org.apache.commons.compress.utils.IOUtils;
--import org.apache.kylin.common.util.BytesUtil;
--
--import com.google.common.hash.HashFunction;
--import com.google.common.hash.Hashing;
- import com.ning.compress.lzf.LZFDecoder;
- import com.ning.compress.lzf.LZFEncoder;
--
--/**
-- * About compression, test on HLLC data shows
-- *
- * - LZF compression ratio is around 65%-80%, fast - GZIP compression ratio is
- * around 41%-46%, very slow
- * - LZF compression ratio is around 65%-80%, fast
- * - GZIP compression ratio is around 41%-46%, very slow
-- *
-- * @author yangli9
-- */
--public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter> {
--
-- private final int p;
-- private final int m;
-- private final HashFunction hashFunc;
-- byte[] registers;
--
-- public HyperLogLogPlusCounter() {
-- this(10);
-- }
--
-- public HyperLogLogPlusCounter(int p) {
-- this(p, Hashing.murmur3_128());
-- }
--
-- public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
-- this(another.p, another.hashFunc);
-- merge(another);
-- }
--
-- /** The larger p is, the more storage (2^p bytes), the better accuracy */
-- private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
-- this.p = p;
- this.m = 1 << p;//(int) Math.pow(2, p);
- this.m = (int) Math.pow(2, p);
-- this.hashFunc = hashFunc;
-- this.registers = new byte[m];
-- }
--
-- public void clear() {
- byte zero = (byte) 0;
- Arrays.fill(registers, zero);
- }
-
- public void add(int value) {
- add(hashFunc.hashInt(value).asLong());
- for (int i = 0; i < m; i++)
- registers[i] = 0;
-- }
--
-- public void add(String value) {
-- add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
-- }
--
-- public void add(byte[] value) {
-- add(hashFunc.hashBytes(value).asLong());
- }
-
- public void add(byte[] value, int offset, int length) {
- add(hashFunc.hashBytes(value, offset, length).asLong());
-- }
--
-- protected void add(long hash) {
-- int bucketMask = m - 1;
-- int bucket = (int) (hash & bucketMask);
-- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
--
-- if (firstOnePos > registers[bucket])
-- registers[bucket] = (byte) firstOnePos;
-- }
--
-- public void merge(HyperLogLogPlusCounter another) {
-- assert this.p == another.p;
-- assert this.hashFunc == another.hashFunc;
--
-- for (int i = 0; i < m; i++) {
-- if (registers[i] < another.registers[i])
-- registers[i] = another.registers[i];
-- }
-- }
--
-- public long getCountEstimate() {
-- return new HLLCSnapshot(this).getCountEstimate();
-- }
--
-- public int getMemBytes() {
-- return 12 + m;
-- }
--
-- public double getErrorRate() {
-- return 1.04 / Math.sqrt(m);
-- }
--
-- private int size() {
-- int size = 0;
-- for (int i = 0; i < m; i++) {
-- if (registers[i] > 0)
-- size++;
-- }
-- return size;
-- }
--
-- // ============================================================================
--
-- // a memory efficient snapshot of HLL registers which can yield count
-- // estimate later
-- public static class HLLCSnapshot {
-- byte p;
-- double registerSum;
-- int zeroBuckets;
--
-- public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
-- p = (byte) hllc.p;
-- registerSum = 0;
-- zeroBuckets = 0;
--
-- byte[] registers = hllc.registers;
-- for (int i = 0; i < hllc.m; i++) {
-- if (registers[i] == 0) {
-- registerSum++;
-- zeroBuckets++;
-- } else {
-- registerSum += 1.0 / (1 << registers[i]);
-- }
-- }
-- }
--
-- public long getCountEstimate() {
-- int m = (int) Math.pow(2, p);
-- double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m));
-- double alphaMM = alpha * m * m;
-- double estimate = alphaMM / registerSum;
--
-- // small cardinality adjustment
-- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
-- estimate = m * Math.log(m * 1.0 / zeroBuckets);
-- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
-- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
-- }
--
-- return Math.round(estimate);
-- }
-- }
--
-- // ============================================================================
-
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
- // disable
- // due to
- // slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
- // lib
- // has
- // a
- // bug
- // at
- // the
- // moment
-
- public void writeCompactRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
--
- public void writeRegisters(final ByteBuffer out) throws IOException {
-- final int indexLen = getRegisterIndexSize();
-- int size = size();
--
-- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
-- byte scheme;
-- if ((indexLen + 1) * size < m)
-- scheme = 0; // map
-- else
-- scheme = 1; // array
-- out.put(scheme);
--
-- if (scheme == 0) { // map scheme
-- BytesUtil.writeVInt(size, out);
-- for (int i = 0; i < m; i++) {
-- if (registers[i] > 0) {
-- BytesUtil.writeUnsigned(i, indexLen, out);
-- out.put(registers[i]);
-- }
-- }
-- } else { // array scheme
-- for (int i = 0; i < m; i++) {
-- out.put(registers[i]);
-- }
-- }
-
- // do compression if needed
- int len = out.position() - startPos;
- if (len < COMPRESSION_THRESHOLD)
- return;
-
- scheme |= COMPRESSION_FLAG;
- byte[] compressed = DEFAULT_COMPRESSOR.compress(out, startPos + 1, len - 1);
- out.position(startPos);
- out.put(scheme);
- BytesUtil.writeVInt(compressed.length, out);
- out.put(compressed);
-- }
--
- public void readCompactRegisters(ByteBuffer in) throws IOException {
- public void readRegisters(ByteBuffer in) throws IOException {
-- byte scheme = in.get();
- if ((scheme & COMPRESSION_FLAG) > 0) {
- scheme ^= COMPRESSION_FLAG;
- int compressedLen = BytesUtil.readVInt(in);
- int end = in.position() + compressedLen;
- byte[] decompressed = DEFAULT_COMPRESSOR.decompress(in, in.position(), compressedLen);
- in.position(end);
- in = ByteBuffer.wrap(decompressed);
- }
--
-- if (scheme == 0) { // map scheme
-- clear();
-- int size = BytesUtil.readVInt(in);
-- if (size > m)
-- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
-- int indexLen = getRegisterIndexSize();
-- for (int i = 0; i < size; i++) {
-- int key = BytesUtil.readUnsigned(in, indexLen);
-- registers[key] = in.get();
-- }
-- } else { // array scheme
- in.get(registers);
- for (int i = 0; i < m; i++) {
- registers[i] = in.get();
- }
-- }
- }
-
- /**
- * For compressed output use writeCompactRegisters
- * @param out
- */
- public void writeRegisters(final ByteBuffer out) {
- out.put(this.registers);
- }
-
- /**
- * For compressed input use readCompactRegisters
- * @param in
- */
- public void readRegisters(ByteBuffer in) {
- in.get(registers, 0, m);
-- }
--
-- private int getRegisterIndexSize() {
-- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
-- }
--
-- @Override
-- public int hashCode() {
-- final int prime = 31;
-- int result = 1;
-- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
-- result = prime * result + p;
-- result = prime * result + Arrays.hashCode(registers);
-- return result;
-- }
--
-- @Override
-- public boolean equals(Object obj) {
-- if (this == obj)
-- return true;
-- if (obj == null)
-- return false;
-- if (getClass() != obj.getClass())
-- return false;
-- HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
-- if (hashFunc == null) {
-- if (other.hashFunc != null)
-- return false;
-- } else if (!hashFunc.equals(other.hashFunc))
-- return false;
-- if (p != other.p)
-- return false;
-- if (!Arrays.equals(registers, other.registers))
-- return false;
-- return true;
-- }
--
-- @Override
-- public int compareTo(HyperLogLogPlusCounter o) {
-- if (o == null)
-- return 1;
--
-- long e1 = this.getCountEstimate();
-- long e2 = o.getCountEstimate();
--
-- if (e1 == e2)
-- return 0;
-- else if (e1 > e2)
-- return 1;
-- else
-- return -1;
-- }
--
-- public static void main(String[] args) throws IOException {
-- dumpErrorRates();
-- }
--
-- static void dumpErrorRates() {
-- for (int p = 10; p <= 18; p++) {
-- double rate = new HyperLogLogPlusCounter(p).getErrorRate();
-- double er = Math.round(rate * 10000) / 100D;
-- double er2 = Math.round(rate * 2 * 10000) / 100D;
-- double er3 = Math.round(rate * 3 * 10000) / 100D;
-- long size = Math.round(Math.pow(2, p));
-- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
-- }
-- }
--}
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++*/
++
++package org.apache.kylin.common.hll;
++
++import java.io.IOException;
++import java.nio.ByteBuffer;
++import java.nio.charset.Charset;
++import java.util.Arrays;
++
++import org.apache.kylin.common.util.BytesUtil;
++
++import com.google.common.hash.HashFunction;
++import com.google.common.hash.Hashing;
++
++/**
++ * About compression, test on HLLC data shows
++ *
++ * - LZF compression ratio is around 65%-80%, fast
++ * - GZIP compression ratio is around 41%-46%, very slow
++ *
++ * @author yangli9
++ */
++public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter> {
++
++ private final int p;
++ private final int m;
++ private final HashFunction hashFunc;
++ byte[] registers;
++
++ public HyperLogLogPlusCounter() {
++ this(10);
++ }
++
++ public HyperLogLogPlusCounter(int p) {
++ this(p, Hashing.murmur3_128());
++ }
++
++ public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
++ this(another.p, another.hashFunc);
++ merge(another);
++ }
++
++ /** The larger p is, the more storage (2^p bytes), the better accuracy */
++ private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
++ this.p = p;
++ this.m = 1 << p;//(int) Math.pow(2, p);
++ this.hashFunc = hashFunc;
++ this.registers = new byte[m];
++ }
++
++ public void clear() {
++ byte zero = (byte) 0;
++ Arrays.fill(registers, zero);
++ }
++
++ public void add(int value) {
++ add(hashFunc.hashInt(value).asLong());
++ }
++
++ public void add(String value) {
++ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
++ }
++
++ public void add(byte[] value) {
++ add(hashFunc.hashBytes(value).asLong());
++ }
++
++ public void add(byte[] value, int offset, int length) {
++ add(hashFunc.hashBytes(value, offset, length).asLong());
++ }
++
++ protected void add(long hash) {
++ int bucketMask = m - 1;
++ int bucket = (int) (hash & bucketMask);
++ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
++
++ if (firstOnePos > registers[bucket])
++ registers[bucket] = (byte) firstOnePos;
++ }
++
++ public void merge(HyperLogLogPlusCounter another) {
++ assert this.p == another.p;
++ assert this.hashFunc == another.hashFunc;
++
++ for (int i = 0; i < m; i++) {
++ if (registers[i] < another.registers[i])
++ registers[i] = another.registers[i];
++ }
++ }
++
++ public long getCountEstimate() {
++ return new HLLCSnapshot(this).getCountEstimate();
++ }
++
++ public int getMemBytes() {
++ return 12 + m;
++ }
++
++ public double getErrorRate() {
++ return 1.04 / Math.sqrt(m);
++ }
++
++ private int size() {
++ int size = 0;
++ for (int i = 0; i < m; i++) {
++ if (registers[i] > 0)
++ size++;
++ }
++ return size;
++ }
++
++ // ============================================================================
++
++ // a memory efficient snapshot of HLL registers which can yield count
++ // estimate later
++ public static class HLLCSnapshot {
++ byte p;
++ double registerSum;
++ int zeroBuckets;
++
++ public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
++ p = (byte) hllc.p;
++ registerSum = 0;
++ zeroBuckets = 0;
++
++ byte[] registers = hllc.registers;
++ for (int i = 0; i < hllc.m; i++) {
++ if (registers[i] == 0) {
++ registerSum++;
++ zeroBuckets++;
++ } else {
++ registerSum += 1.0 / (1 << registers[i]);
++ }
++ }
++ }
++
++ public long getCountEstimate() {
++ int m = (int) Math.pow(2, p);
++ double alpha = 1 / (2 * Math.log(2) * (1 + (3 * Math.log(2) - 1) / m));
++ double alphaMM = alpha * m * m;
++ double estimate = alphaMM / registerSum;
++
++ // small cardinality adjustment
++ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
++ estimate = m * Math.log(m * 1.0 / zeroBuckets);
++ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
++ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
++ }
++
++ return Math.round(estimate);
++ }
++ }
++
++ // ============================================================================
++
++ public static interface Compressor {
++
++ byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
++
++ byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
++ }
++
++ static final Compressor GZIP_COMPRESSOR = new Compressor() {
++ @Override
++ public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
++ ByteArrayOutputStream bout = new ByteArrayOutputStream();
++ GZIPOutputStream gzout = new GZIPOutputStream(bout);
++ gzout.write(buf.array(), offset, length);
++ gzout.close();
++ return bout.toByteArray();
++ }
++
++ @Override
++ public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
++ ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
++ GZIPInputStream gzin = new GZIPInputStream(bin);
++ ByteArrayOutputStream bout = new ByteArrayOutputStream();
++ IOUtils.copy(gzin, bout);
++ gzin.close();
++ bout.close();
++ return bout.toByteArray();
++ }
++ };
++
++ static final Compressor LZF_COMPRESSOR = new Compressor() {
++ @Override
++ public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
++ return LZFEncoder.encode(buf.array(), offset, length);
++ }
++
++ @Override
++ public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
++ return LZFDecoder.decode(buf.array(), offset, length);
++ }
++ };
++
++ public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
++ // disable
++ // due to
++ // slowness
++ public static final byte COMPRESSION_FLAG = (byte) 0x02;
++ public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
++ // lib
++ // has
++ // a
++ // bug
++ // at
++ // the
++ // moment
++
++ public void writeCompactRegisters(final ByteBuffer out) throws IOException {
++ int startPos = out.position();
++
++ final int indexLen = getRegisterIndexSize();
++ int size = size();
++
++ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
++ byte scheme;
++ if ((indexLen + 1) * size < m)
++ scheme = 0; // map
++ else
++ scheme = 1; // array
++ out.put(scheme);
++
++ if (scheme == 0) { // map scheme
++ BytesUtil.writeVInt(size, out);
++ for (int i = 0; i < m; i++) {
++ if (registers[i] > 0) {
++ BytesUtil.writeUnsigned(i, indexLen, out);
++ out.put(registers[i]);
++ }
++ }
++ } else { // array scheme
++ for (int i = 0; i < m; i++) {
++ out.put(registers[i]);
++ }
++ }
++ }
++
++ public void readCompactRegisters(ByteBuffer in) throws IOException {
++ byte scheme = in.get();
++
++ if (scheme == 0) { // map scheme
++ clear();
++ int size = BytesUtil.readVInt(in);
++ if (size > m)
++ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
++ int indexLen = getRegisterIndexSize();
++ for (int i = 0; i < size; i++) {
++ int key = BytesUtil.readUnsigned(in, indexLen);
++ registers[key] = in.get();
++ }
++ } else { // array scheme
++ in.get(registers);
++ }
++ }
++
++ /**
++ * For compressed output use writeCompactRegisters
++ * @param out
++ */
++ public void writeRegisters(final ByteBuffer out) {
++ out.put(this.registers);
++ }
++
++ /**
++ * For compressed input use readCompactRegisters
++ * @param in
++ */
++ public void readRegisters(ByteBuffer in) {
++ in.get(registers, 0, m);
++ }
++
++ private int getRegisterIndexSize() {
++ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
++ }
++
++ @Override
++ public int hashCode() {
++ final int prime = 31;
++ int result = 1;
++ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
++ result = prime * result + p;
++ result = prime * result + Arrays.hashCode(registers);
++ return result;
++ }
++
++ @Override
++ public boolean equals(Object obj) {
++ if (this == obj)
++ return true;
++ if (obj == null)
++ return false;
++ if (getClass() != obj.getClass())
++ return false;
++ HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
++ if (hashFunc == null) {
++ if (other.hashFunc != null)
++ return false;
++ } else if (!hashFunc.equals(other.hashFunc))
++ return false;
++ if (p != other.p)
++ return false;
++ if (!Arrays.equals(registers, other.registers))
++ return false;
++ return true;
++ }
++
++ @Override
++ public int compareTo(HyperLogLogPlusCounter o) {
++ if (o == null)
++ return 1;
++
++ long e1 = this.getCountEstimate();
++ long e2 = o.getCountEstimate();
++
++ if (e1 == e2)
++ return 0;
++ else if (e1 > e2)
++ return 1;
++ else
++ return -1;
++ }
++
++ public static void main(String[] args) throws IOException {
++ dumpErrorRates();
++ }
++
++ static void dumpErrorRates() {
++ for (int p = 10; p <= 18; p++) {
++ double rate = new HyperLogLogPlusCounter(p).getErrorRate();
++ double er = Math.round(rate * 10000) / 100D;
++ double er2 = Math.round(rate * 2 * 10000) / 100D;
++ double er3 = Math.round(rate * 3 * 10000) / 100D;
++ long size = Math.round(Math.pow(2, p));
++ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
++ }
++ }
++}
[13/50] incubator-kylin git commit: fix .gitignore
Posted by li...@apache.org.
fix .gitignore
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cb1de27c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cb1de27c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cb1de27c
Branch: refs/heads/inverted-index
Commit: cb1de27c14103e16dba7f33af03fdff1ada996a5
Parents: c94519a
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:30:16 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:30:16 2015 +0800
----------------------------------------------------------------------
.gitignore | 1 -
bin/check-env.sh | 39 ++++++++++++++++++++++++++
bin/find-hive-dependency.sh | 20 ++++++++++++++
bin/healthmon.sh | 33 ++++++++++++++++++++++
bin/start-kylin.sh | 59 ++++++++++++++++++++++++++++++++++++++++
bin/stop-kylin.sh | 4 +++
6 files changed, 155 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 4d4bf12..6649d69 100755
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,6 @@
*.pydevproject
.metadata
.gradle
-bin/
tmp/
*.tmp
*.bak
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/bin/check-env.sh b/bin/check-env.sh
new file mode 100755
index 0000000..3bef6c4
--- /dev/null
+++ b/bin/check-env.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+echo "Checking KYLIN_HOME..."
+if [ -z "$KYLIN_HOME" ]
+then
+ echo 'please make sure KYLIN_HOME has been set'
+ exit 1
+else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+fi
+
+echo "Checking hbase..."
+if [ -z "$(command -v hbase version)" ]
+then
+ echo "Please make sure the user has the privilege to run hbase shell"
+ exit 1
+else
+ echo "hbase check passed"
+fi
+
+echo "Checking hive..."
+if [ -z "$(command -v hive --version)" ]
+then
+ echo "Please make sure the user has the privilege to run hive shell"
+ exit 1
+else
+ echo "hive check passed"
+fi
+
+echo "Checking hadoop..."
+if [ -z "$(command -v hadoop version)" ]
+then
+ echo "Please make sure the user has the privilege to run hadoop shell"
+ exit 1
+else
+ echo "hadoop check passed"
+fi
+
+exit 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
new file mode 100644
index 0000000..7bd5f43
--- /dev/null
+++ b/bin/find-hive-dependency.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+hive_env=`hive -e set | grep 'env:CLASSPATH'`
+
+hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
+arr=(`echo $hive_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hive_exec_path=
+for data in ${arr[@]}
+do
+ result=`echo $data | grep 'hive-exec.jar'`
+ if [ $result ]
+ then
+ hive_exec_path=$data
+ fi
+done
+hdp_home=`echo $hive_exec_path | awk -F '/hive/lib/' '{print $1}'`
+
+hive_dependency=/usr/hdp/current/hive-client/conf/:${hdp_home}/hive/lib/*:${hdp_home}/hive-hcatalog/share/hcatalog/*
+echo "hive dependency: $hive_dependency"
+export hive_dependency
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/bin/healthmon.sh
----------------------------------------------------------------------
diff --git a/bin/healthmon.sh b/bin/healthmon.sh
new file mode 100644
index 0000000..439eef4
--- /dev/null
+++ b/bin/healthmon.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+ALERT="your@email.com"
+
+OUTPUT=$(
+ curl --max-time 20 -# \
+ --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
+ -H "Authorization:Basic QURNSU46S1lMSU4=" \
+ -H "Content-Type:application/json;charset=UTF-8" \
+ http://localhost:7070/kylin/api/query \
+)
+
+# ----------------------------------------------------------------------------
+
+date
+
+if [[ $OUTPUT == *"results"* ]]; then
+ echo "Good."
+else
+ echo "Bad."
+ TS_FILE=/tmp/kylin_healthmon_ts
+ LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
+ CURR_TS=`date +%s`
+ echo last: $LAST_TS
+ echo curr: $CURR_TS
+ if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
+ echo "Sending mail..."
+ echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
+ if [ "$?" == "0" ]; then
+ touch $TS_FILE
+ fi
+ fi
+fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
new file mode 100755
index 0000000..d328851
--- /dev/null
+++ b/bin/start-kylin.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+
+dir=$(dirname ${0})
+
+tomcat_root=${dir}/../tomcat
+export tomcat_root
+
+sh ${dir}/check-env.sh || { exit 1; }
+
+#if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
+#then
+# echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
+#else
+# exit 1
+#fi
+
+#The location of all hadoop/hbase configurations are difficult to get.
+#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
+#For example, in hdp 2.2, there is a system property called hdp.version,
+#which we cannot get until running hbase or hadoop shell command.
+#
+#To save all these troubles, we use hbase runjar to start tomcat.
+#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
+#hbase command will do all the dirty tasks for us:
+
+#-Djava.library.path=${KYLIN_LD_LIBRARY_PATH} \
+
+
+useSandbox=`cat ${KYLIN_HOME}/conf/kylin.properties | grep 'kylin.sandbox' | awk -F '=' '{print $2}'`
+spring_profile="default"
+if [ "$useSandbox" -eq true ]
+ then spring_profile="sandbox"
+fi
+rm -rf ${tomcat_root}/webapps/kylin*
+cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
+
+source ${dir}/find-hive-dependency.sh
+
+export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
+export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
+-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
+-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
+-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
+-Djava.endorsed.dirs=${tomcat_root}/endorsed \
+-Dcatalina.base=${tomcat_root} \
+-Dcatalina.home=${tomcat_root} \
+-Djava.io.tmpdir=${tomcat_root}/temp \
+-Dkylin.hive.dependency=${hive_dependency} \
+-Dspring.profiles.active=${spring_profile} \
+org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
+echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
+if [ "$useSandbox" -eq true ]
+ then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
+else
+ echo "Please visit http://<ip>:7070/kylin"
+fi
+echo "You can check the log at ${tomcat_root}/logs/kylin_sandbox.log"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cb1de27c/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
new file mode 100755
index 0000000..806513b
--- /dev/null
+++ b/bin/stop-kylin.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+ps -fu $USER | grep tomcat | grep -v "grep" | awk '{print $2}' | xargs kill
+echo "all tomcats started by $USER are killed"
\ No newline at end of file
[15/50] incubator-kylin git commit: Update unit tests
Posted by li...@apache.org.
Update unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a7b4ac81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a7b4ac81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a7b4ac81
Branch: refs/heads/inverted-index
Commit: a7b4ac81f05e664b56585a5e1db6cfb75dd13589
Parents: 7eb78db
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Tue Feb 10 17:38:56 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Tue Feb 10 17:38:56 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CacheController.java | 13 ++-
.../kylin/rest/controller/CubeController.java | 115 +++++++++----------
.../kylin/rest/controller/ModelController.java | 6 +-
.../kylin/rest/controller/TableController.java | 46 ++++----
.../apache/kylin/rest/service/CubeService.java | 79 ++++++-------
.../rest/controller/CacheControllerTest.java | 57 +++++++++
.../rest/controller/CubeControllerTest.java | 47 ++++++--
.../rest/controller/TableControllerTest.java | 74 ++++++++++++
.../kylin/rest/service/ServiceTestBase.java | 22 ++--
9 files changed, 305 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index cb7f40f..42f5ad8 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -36,7 +36,6 @@ import java.io.IOException;
* CubeController is defined as Restful API entrance for UI.
*
* @author jianliu
- *
*/
@Controller
@RequestMapping(value = "/cache")
@@ -49,15 +48,13 @@ public class CacheController extends BasicController {
/**
* Wipe system cache
*
- * @param type
- * {@link Broadcaster.TYPE}
- * @param event
- * {@link Broadcaster.EVENT}
+ * @param type {@link Broadcaster.TYPE}
+ * @param event {@link Broadcaster.EVENT}
* @param name
* @return if the action success
* @throws IOException
*/
- @RequestMapping(value = "/{type}/{name}/{event}", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{type}/{name}/{event}", method = {RequestMethod.PUT})
@ResponseBody
public void wipeCache(@PathVariable String type, @PathVariable String event, @PathVariable String name) throws IOException {
Broadcaster.TYPE wipeType = Broadcaster.TYPE.getType(type);
@@ -76,4 +73,8 @@ public class CacheController extends BasicController {
throw new RuntimeException("invalid type:" + wipeEvent);
}
}
+
+ public void setCacheService(CacheService cacheService) {
+ this.cacheService = cacheService;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 1733a1a..e624d45 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -18,40 +18,11 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.exception.ForbiddenException;
-import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.request.JobBuildRequest;
-import org.apache.kylin.rest.response.GeneralResponse;
-import org.apache.kylin.rest.response.HBaseResponse;
-import org.apache.kylin.rest.service.CubeService;
-import org.quartz.SchedulerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.security.access.AccessDeniedException;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
import com.codahale.metrics.annotation.Metered;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
@@ -66,14 +37,33 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.request.JobBuildRequest;
+import org.apache.kylin.rest.response.GeneralResponse;
+import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.quartz.SchedulerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.*;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.*;
/**
* CubeController is defined as Restful API entrance for UI.
- *
+ *
* @author jianliu
*/
@Controller
@@ -87,7 +77,7 @@ public class CubeController extends BasicController {
@Autowired
private JobService jobService;
- @RequestMapping(value = "", method = { RequestMethod.GET })
+ @RequestMapping(value = "", method = {RequestMethod.GET})
@ResponseBody
@Metered(name = "listCubes")
public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam("limit") Integer limit, @RequestParam("offset") Integer offset) {
@@ -96,14 +86,13 @@ public class CubeController extends BasicController {
/**
* Get hive SQL of the cube
- *
- * @param cubeName
- * Cube Name
+ *
+ * @param cubeName Cube Name
* @return
* @throws UnknownHostException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = {RequestMethod.GET})
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -120,13 +109,13 @@ public class CubeController extends BasicController {
/**
* Update cube notify list
- *
+ *
* @param cubeName
* @param notifyList
* @throws IOException
* @throws CubeIntegrityException
*/
- @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/notify_list", method = {RequestMethod.PUT})
@ResponseBody
public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -144,7 +133,7 @@ public class CubeController extends BasicController {
}
- @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/cost", method = {RequestMethod.PUT})
@ResponseBody
@Metered(name = "updateCubeCost")
public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
@@ -157,7 +146,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = {RequestMethod.PUT})
@ResponseBody
public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
try {
@@ -172,10 +161,10 @@ public class CubeController extends BasicController {
/**
* Force rebuild a cube's lookup table snapshot
- *
+ *
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = {RequestMethod.PUT})
@ResponseBody
public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
try {
@@ -188,14 +177,13 @@ public class CubeController extends BasicController {
/**
* Send a rebuild cube job
- *
- * @param cubeName
- * Cube ID
+ *
+ * @param cubeName Cube ID
* @return
* @throws SchedulerException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest jobBuildRequest) {
try {
@@ -212,7 +200,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/disable", method = {RequestMethod.PUT})
@ResponseBody
@Metered(name = "disableCube")
public CubeInstance disableCube(@PathVariable String cubeName) {
@@ -231,7 +219,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/purge", method = {RequestMethod.PUT})
@ResponseBody
@Metered(name = "purgeCube")
public CubeInstance purgeCube(@PathVariable String cubeName) {
@@ -250,7 +238,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{cubeName}/enable", method = {RequestMethod.PUT})
@ResponseBody
@Metered(name = "enableCube")
public CubeInstance enableCube(@PathVariable String cubeName) {
@@ -268,7 +256,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
+ @RequestMapping(value = "/{cubeName}", method = {RequestMethod.DELETE})
@ResponseBody
@Metered(name = "deleteCube")
public void deleteCube(@PathVariable String cubeName) {
@@ -287,22 +275,29 @@ public class CubeController extends BasicController {
/**
* Get available table list of the input database
- *
+ *
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = { RequestMethod.POST })
+ @RequestMapping(value = "", method = {RequestMethod.POST})
@ResponseBody
@Metered(name = "saveCube")
public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
//Update Model
MetadataManager metaManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc modelDesc = deserializeDataModelDesc(cubeRequest);
- if (modelDesc == null) {
+ if (modelDesc == null || StringUtils.isEmpty(modelDesc.getName())) {
return cubeRequest;
}
+
try {
- metaManager.createDataModelDesc(modelDesc);
+ DataModelDesc existingModel = metaManager.getDataModelDesc(modelDesc.getName());
+ if (existingModel == null) {
+ metaManager.createDataModelDesc(modelDesc);
+ } else {
+ modelDesc.setLastModified(existingModel.getLastModified());
+ metaManager.updateDataModelDesc(modelDesc);
+ }
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
@@ -336,12 +331,12 @@ public class CubeController extends BasicController {
/**
* Get available table list of the input database
- *
+ *
* @return Table metadata array
- * @throws JsonProcessingException
+ * @throws JsonProcessingException
* @throws IOException
*/
- @RequestMapping(value = "", method = { RequestMethod.PUT })
+ @RequestMapping(value = "", method = {RequestMethod.PUT})
@ResponseBody
@Metered(name = "updateCube")
public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
@@ -400,11 +395,11 @@ public class CubeController extends BasicController {
/**
* Get available table list of the input database
- *
+ *
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
+ @RequestMapping(value = "/{cubeName}/hbase", method = {RequestMethod.GET})
@ResponseBody
@Metered(name = "getHBaseInfo")
public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
@@ -481,7 +476,7 @@ public class CubeController extends BasicController {
*/
private String omitMessage(List<String> errors) {
StringBuffer buffer = new StringBuffer();
- for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext(); ) {
String string = (String) iterator.next();
buffer.append(string);
buffer.append("\n");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/main/java/org/apache/kylin/rest/controller/ModelController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 326fd53..2499f59 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -53,11 +53,15 @@ public class ModelController {
*/
@RequestMapping(value = "/{model_name}", method = { RequestMethod.GET })
@ResponseBody
- public DataModelDesc getCube(@PathVariable String model_name) {
+ public DataModelDesc getModel(@PathVariable String model_name) {
MetadataManager metaManager= MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc modeDesc = metaManager.getDataModelDesc(model_name);
return modeDesc;
}
+ public void setCubeService(CubeService cubeService) {
+ this.cubeService = cubeService;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 30f81b5..99b0875 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -18,35 +18,24 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
+import com.codahale.metrics.annotation.Metered;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.CardinalityRequest;
+import org.apache.kylin.rest.response.TableDescResponse;
import org.apache.kylin.rest.service.CubeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
-import com.codahale.metrics.annotation.Metered;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.rest.request.CardinalityRequest;
-import org.apache.kylin.rest.response.TableDescResponse;
+import java.io.IOException;
+import java.util.*;
/**
* @author xduo
@@ -65,7 +54,7 @@ public class TableController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = { RequestMethod.GET })
+ @RequestMapping(value = "", method = {RequestMethod.GET})
@ResponseBody
@Metered(name = "listSourceTables")
public List<TableDesc> getHiveTables(@RequestParam(value = "ext", required = false) boolean withExt, @RequestParam(value = "project", required = false) String project) {
@@ -93,7 +82,7 @@ public class TableController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "/{tableName}", method = { RequestMethod.GET })
+ @RequestMapping(value = "/{tableName}", method = {RequestMethod.GET})
@ResponseBody
public TableDesc getHiveTable(@PathVariable String tableName) {
return cubeMgmtService.getMetadataManager().getTableDesc(tableName);
@@ -105,21 +94,21 @@ public class TableController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "/{tableName}/exd-map", method = { RequestMethod.GET })
+ @RequestMapping(value = "/{tableName}/exd-map", method = {RequestMethod.GET})
@ResponseBody
public Map<String, String> getHiveTableExd(@PathVariable String tableName) {
Map<String, String> tableExd = cubeMgmtService.getMetadataManager().getTableDescExd(tableName);
return tableExd;
}
- @RequestMapping(value = "/reload", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/reload", method = {RequestMethod.PUT})
@ResponseBody
public String reloadSourceTable() {
cubeMgmtService.getMetadataManager().reload();
return "ok";
}
- @RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.POST })
+ @RequestMapping(value = "/{tables}/{project}", method = {RequestMethod.POST})
@ResponseBody
public Map<String, String[]> loadHiveTable(@PathVariable String tables, @PathVariable String project) throws IOException {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
@@ -138,7 +127,7 @@ public class TableController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "/{tableNames}/cardinality", method = { RequestMethod.PUT })
+ @RequestMapping(value = "/{tableNames}/cardinality", method = {RequestMethod.PUT})
@ResponseBody
public CardinalityRequest generateCardinality(@PathVariable String tableNames, @RequestBody CardinalityRequest request) {
String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
@@ -192,4 +181,9 @@ public class TableController extends BasicController {
}
return descs;
}
+
+ public void setCubeService(CubeService cubeService) {
+ this.cubeMgmtService = cubeService;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d9373ca..de97a7b 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -18,32 +18,8 @@
package org.apache.kylin.rest.service;
-import java.io.IOException;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTable;
-import org.apache.kylin.rest.controller.QueryController;
-import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.response.MetricsResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.cache.annotation.CacheEvict;
-import org.springframework.cache.annotation.Caching;
-import org.springframework.security.access.prepost.PostFilter;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.stereotype.Component;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseRegionSizeCalculator;
import org.apache.kylin.common.util.HadoopUtil;
@@ -70,9 +46,25 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.tool.HiveSourceTableLoader;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.controller.QueryController;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Caching;
+import org.springframework.security.access.prepost.PostFilter;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.*;
/**
* Stateless & lightweight service facade of cube management functions.
@@ -156,10 +148,19 @@ public class CubeService extends BasicService {
CubeDesc createdDesc = null;
CubeInstance createdCube = null;
- createdDesc = getCubeDescManager().createCubeDesc(desc);
+ boolean isNew = false;
+ if (getCubeDescManager().getCubeDesc(desc.getName()) == null) {
+ createdDesc = getCubeDescManager().createCubeDesc(desc);
+ isNew = true;
+ } else {
+ createdDesc = getCubeDescManager().updateCubeDesc(desc);
+ }
+
if (!createdDesc.getError().isEmpty()) {
- getCubeDescManager().removeCubeDesc(createdDesc);
+ if (isNew) {
+ getCubeDescManager().removeCubeDesc(createdDesc);
+ }
throw new InternalErrorException(createdDesc.getError().get(0));
}
@@ -305,7 +306,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
+ @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -332,7 +333,7 @@ public class CubeService extends BasicService {
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
- @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
+ @Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
@@ -471,27 +472,27 @@ public class CubeService extends BasicService {
logger.error("Cannot find table descirptor " + tableName, e);
throw e;
}
-
+
DefaultChainedExecutable job = new DefaultChainedExecutable();
job.setName("Hive Column Cardinality calculation for table '" + tableName + "'");
job.setSubmitter(submitter);
String outPath = HiveColumnCardinalityJob.OUTPUT_PATH + "/" + tableName;
String param = "-table " + tableName + " -output " + outPath;
-
+
HadoopShellExecutable step1 = new HadoopShellExecutable();
-
+
step1.setJobClass(HiveColumnCardinalityJob.class);
step1.setJobParams(param);
-
+
job.addTask(step1);
-
+
HadoopShellExecutable step2 = new HadoopShellExecutable();
-
+
step2.setJobClass(HiveColumnCardinalityUpdateJob.class);
step2.setJobParams(param);
job.addTask(step2);
-
+
getExecutableManager().addJob(job);
}
@@ -531,7 +532,7 @@ public class CubeService extends BasicService {
CubeManager.getInstance(getConfig()).updateCube(cube);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
public String[] reloadHiveTable(String tables) throws IOException {
Set<String> loaded = HiveSourceTableLoader.reloadHiveTables(tables.split(","), getConfig());
return (String[]) loaded.toArray(new String[loaded.size()]);
@@ -541,9 +542,9 @@ public class CubeService extends BasicService {
public void syncTableToProject(String[] tables, String project) throws IOException {
getProjectManager().addTableDescToProject(tables, project);
}
-
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER)
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException {
MetadataManager metaMgr = getMetadataManager();
for (String table : tables) {
@@ -554,5 +555,5 @@ public class CubeService extends BasicService {
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
new file mode 100644
index 0000000..d75c5ca
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.service.CacheService;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shaoshi
+ */
+public class CacheControllerTest extends ServiceTestBase {
+
+ private CacheController cacheController;
+
+ @Autowired
+ private CacheService cacheService;
+
+ @Before
+ public void setup() throws Exception {
+ super.setUp();
+
+ cacheController = new CacheController();
+ cacheController.setCacheService(cacheService);
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+
+ cacheController.wipeCache("cube_desc", "drop", "test_kylin_cube_with_slr_desc");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 52247d3..ee57cd3 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -18,20 +18,23 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.io.StringWriter;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
/**
* @author xduo
@@ -40,6 +43,7 @@ public class CubeControllerTest extends ServiceTestBase {
private CubeController cubeController;
private CubeDescController cubeDescController;
+ private ModelController modelController;
@Autowired
CubeService cubeService;
@@ -55,6 +59,9 @@ public class CubeControllerTest extends ServiceTestBase {
cubeController.setJobService(jobService);
cubeDescController = new CubeDescController();
cubeDescController.setCubeService(cubeService);
+
+ modelController = new ModelController();
+ modelController.setCubeService(cubeService);
}
@Test
@@ -67,6 +74,13 @@ public class CubeControllerTest extends ServiceTestBase {
CubeDesc cube = cubes[0];
CubeDesc newCube = new CubeDesc();
String newCubeName = cube.getName() + "_test_save";
+
+ try {
+ cubeController.deleteCube(newCubeName);
+ } catch (Exception e) {
+ // it may not exist, ignore the exception
+ }
+
newCube.setName(newCubeName);
newCube.setModelName(cube.getModelName());
newCube.setModel(cube.getModel());
@@ -76,7 +90,8 @@ public class CubeControllerTest extends ServiceTestBase {
newCube.setConfig(cube.getConfig());
newCube.setRowkey(cube.getRowkey());
- newCube.getModel().setName(newCubeName + "_model_desc" + System.currentTimeMillis());//generate a random model
+ String newModelName = newCubeName + "_model_desc";
+ newCube.getModel().setName(newModelName);//generate a random model
newCube.getModel().setLastModified(0);
ObjectMapper cubeDescMapper = new ObjectMapper();
@@ -92,6 +107,20 @@ public class CubeControllerTest extends ServiceTestBase {
cubeRequest.setModelDescData(modelDescWriter.toString());
cubeRequest = cubeController.saveCubeDesc(cubeRequest);
+
+ DataModelDesc model = modelController.getModel(newModelName);
+ Assert.assertNotNull(model);
+
+ List<String> notifyList = Lists.newArrayList();
+ notifyList.add("john@example.com");
+ cubeController.updateNotifyList(newCubeName, notifyList);
+ cubeController.updateCubeCost(newCubeName, 80);
+
+ List<CubeInstance> cubeInstances = cubeController.getCubes(newCubeName, "default", 1, 0);
+
+ CubeInstance cubeInstance = cubeInstances.get(0);
+ Assert.assertTrue(cubeInstance.getDescriptor().getNotifyList().contains("john@example.com"));
+ Assert.assertTrue(cubeInstance.getCost() == 80);
cubeController.deleteCube(newCubeName);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
new file mode 100644
index 0000000..37e184f
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/controller/TableControllerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ServiceTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shaoshi
+ */
+public class TableControllerTest extends ServiceTestBase {
+
+ private TableController tableController;
+ private CubeDescController cubeDescController;
+
+ @Autowired
+ CubeService cubeService;
+
+ @Before
+ public void setup() throws Exception {
+ super.setUp();
+
+ tableController = new TableController();
+ tableController.setCubeService(cubeService);
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ List<TableDesc> tables = tableController.getHiveTables(true, "default");
+
+ Assert.assertTrue(tables != null && tables.size() > 0);
+
+ TableDesc factTable = null;
+ for (TableDesc t : tables) {
+ if (t.getName().equalsIgnoreCase("test_kylin_fact")) {
+ factTable = t;
+ break;
+ }
+ }
+
+ Assert.assertNotNull(factTable);
+
+
+ Map<String, String[]> loadResult = tableController.loadHiveTable("test_kylin_fact,TEST_CATEGORY_GROUPINGS", "default");
+ Assert.assertNotNull(loadResult);
+
+ Assert.assertTrue(loadResult.get("result.loaded").length ==2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a7b4ac81/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 3b923ad..5f98767 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -18,11 +18,13 @@
package org.apache.kylin.rest.service;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.apache.kylin.common.util.HBaseMetadataTestCase;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.invertedindex.IIManager;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.junit.*;
import org.junit.runner.RunWith;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
@@ -30,20 +32,14 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-import org.apache.kylin.common.util.HBaseMetadataTestCase;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.invertedindex.IIManager;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.project.ProjectManager;
/**
* @author xduo
*/
@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = { "classpath:applicationContext.xml", "classpath:kylinSecurity.xml" })
+@ContextConfiguration(locations = {"classpath:applicationContext.xml", "classpath:kylinSecurity.xml"})
@ActiveProfiles("testing")
-public class ServiceTestBase extends HBaseMetadataTestCase { //HBaseMetadataTestCase {
+public class ServiceTestBase extends HBaseMetadataTestCase {
@BeforeClass
public static void setupResource() throws Exception {
[50/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://git-wip-us.apache.org/repos/asf/incubator-kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://git-wip-us.apache.org/repos/asf/incubator-kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/355a09b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/355a09b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/355a09b5
Branch: refs/heads/inverted-index
Commit: 355a09b582aad86453682e9f7f20250a79722ad3
Parents: ed729ee 4056fc3
Author: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Authored: Thu Feb 12 05:34:36 2015 +0000
Committer: liyang@apache.org <ya...@D-SHC-00801746.corp.ebay.com>
Committed: Thu Feb 12 05:34:36 2015 +0000
----------------------------------------------------------------------
----------------------------------------------------------------------
[08/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Conflicts:
sbin/kylin.sh
sbin/start-kylin-sandbox.sh
sbin/start-kylin.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6e53c4d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6e53c4d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6e53c4d4
Branch: refs/heads/inverted-index
Commit: 6e53c4d4bda1179838ae46fe07aec97b3da7f0f8
Parents: 5038abe 9fb8aaa
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 16:51:14 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 16:51:14 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/jdbc/JDBCDriverTest.java | 12 ++-----
webapp/app/js/controllers/page.js | 1 -
webapp/app/js/controllers/projects.js | 3 +-
webapp/app/js/controllers/sourceMeta.js | 13 ++-----
webapp/app/js/model/projectModel.js | 2 ++
webapp/app/js/model/tableModel.js | 7 +++-
webapp/app/partials/cubes/cube_json_edit.html | 2 +-
webapp/app/partials/jobs/job_submit.html | 38 ++++++++++----------
webapp/bower.json | 5 ++-
9 files changed, 39 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
[41/50] incubator-kylin git commit: add installation guide
Posted by li...@apache.org.
add installation guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/56ba5131
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/56ba5131
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/56ba5131
Branch: refs/heads/inverted-index
Commit: 56ba5131a780ce3ada62c9068efab49b67e679b1
Parents: 443b192
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Feb 12 10:35:46 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Feb 12 10:35:46 2015 +0800
----------------------------------------------------------------------
bin/health-check.sh | 33 ++++++++++
bin/healthmon.sh | 33 ----------
.../org/apache/kylin/common/KylinConfig.java | 3 +-
conf/kylin.properties | 8 +--
docs/Installation/install_and_run.md | 69 ++++++++++++++++++++
.../test_case_data/localmeta/kylin.properties | 6 --
.../localmeta_v1/kylin.properties | 6 --
.../minicluster/b-kylin/kylin.properties | 6 --
.../minicluster/b-kylin/meta/kylin.properties | 6 --
.../test_case_data/minicluster/kylin.properties | 6 --
.../sandbox-hdp21/kylin.properties | 6 --
.../test_case_data/sandbox/kylin.properties | 6 --
server/ServiceConfig.md | 4 +-
13 files changed, 105 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/health-check.sh
----------------------------------------------------------------------
diff --git a/bin/health-check.sh b/bin/health-check.sh
new file mode 100644
index 0000000..439eef4
--- /dev/null
+++ b/bin/health-check.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+ALERT="your@email.com"
+
+OUTPUT=$(
+ curl --max-time 20 -# \
+ --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
+ -H "Authorization:Basic QURNSU46S1lMSU4=" \
+ -H "Content-Type:application/json;charset=UTF-8" \
+ http://localhost:7070/kylin/api/query \
+)
+
+# ----------------------------------------------------------------------------
+
+date
+
+if [[ $OUTPUT == *"results"* ]]; then
+ echo "Good."
+else
+ echo "Bad."
+ TS_FILE=/tmp/kylin_healthmon_ts
+ LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
+ CURR_TS=`date +%s`
+ echo last: $LAST_TS
+ echo curr: $CURR_TS
+ if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
+ echo "Sending mail..."
+ echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
+ if [ "$?" == "0" ]; then
+ touch $TS_FILE
+ fi
+ fi
+fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/healthmon.sh
----------------------------------------------------------------------
diff --git a/bin/healthmon.sh b/bin/healthmon.sh
deleted file mode 100644
index 439eef4..0000000
--- a/bin/healthmon.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-ALERT="your@email.com"
-
-OUTPUT=$(
- curl --max-time 20 -# \
- --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
- -H "Authorization:Basic QURNSU46S1lMSU4=" \
- -H "Content-Type:application/json;charset=UTF-8" \
- http://localhost:7070/kylin/api/query \
-)
-
-# ----------------------------------------------------------------------------
-
-date
-
-if [[ $OUTPUT == *"results"* ]]; then
- echo "Good."
-else
- echo "Bad."
- TS_FILE=/tmp/kylin_healthmon_ts
- LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
- CURR_TS=`date +%s`
- echo last: $LAST_TS
- echo curr: $CURR_TS
- if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
- echo "Sending mail..."
- echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
- if [ "$?" == "0" ]; then
- touch $TS_FILE
- fi
- fi
-fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index ef2c6e2..add87e0 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -89,9 +89,8 @@ public class KylinConfig {
public static final String KYLIN_JOB_JAR = "kylin.job.jar";
public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
- public static final String KYLIN_JOB_JAR_LOCAL = "kylin.job.jar.local";
+ public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index bf86a26..f607dca 100755
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -4,20 +4,14 @@
kylin.rest.servers=
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata_qa@hbase
+kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/docs/Installation/install_and_run.md
----------------------------------------------------------------------
diff --git a/docs/Installation/install_and_run.md b/docs/Installation/install_and_run.md
new file mode 100644
index 0000000..d634b32
--- /dev/null
+++ b/docs/Installation/install_and_run.md
@@ -0,0 +1,69 @@
+##Install and Run
+
+###How to run
+
+1. Download the release version(according to the hadoop distribution)
+2. Setup a KYLIN_HOME pointing to the corresponding directory where you extract the release tar
+3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues.
+4. To start Kylin, simply run **bin/start-kylin.sh**
+5. To stop Kylin, simply run **bin/stop-kylin.sh**
+
+
+If you are running Kylin in a cluster or you have multiple Kylin instances, please make sure you have the following property correctly configured.
+
+1. kylin.rest.servers
+
+ List of web servers in use, this enables one web server instance to sync up with other servers.
+
+
+2. kylin.server.mode
+
+ Make sure there is only one instance whose "kylin.server.mode" is set to "all" if there are multiple instances.
+
+
+###The directory structure of the installation
+
+> ├── bin
+> │ ├── check-env.sh
+> │ ├── find-hive-dependency.sh
+> │ ├── health-check.sh
+> │ ├── start-kylin.sh
+> │ └── stop-kylin.sh
+> │
+> ├── conf
+> │ ├── kylin_job_conf.xml
+> │ └── kylin.properties
+> │
+> ├── tomcat
+> │ ├── webapps
+> │ │ └── kylin.war
+> │ …….
+> │
+> └── lib
+> ├── kylin-coprocessor-${version}.jar
+> └── kylin-job-${version}.jar
+
+
+
+###Build from source
+> git clone https://github.com/KylinOLAP/Kylin.git
+> cd KylinOLAP/Kylin
+> sh script/package.sh
+
+In order to generate binary package, **maven** and **npm** are pre-requisites.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 53d24ee..aa18287 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-coprocessor-latest.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta_v1/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta_v1/kylin.properties b/examples/test_case_data/localmeta_v1/kylin.properties
index 96439b4..aa18287 100644
--- a/examples/test_case_data/localmeta_v1/kylin.properties
+++ b/examples/test_case_data/localmeta_v1/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties
index 19c8de6..5d1b667 100644
--- a/examples/test_case_data/minicluster/b-kylin/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
index 419b9e8..17e5c99 100644
--- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index 7816cad..373186c 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:minicluster:/hbase-unsecure
# The storage for final cube file in hbase
kylin.storage.url=hbase:minicluster:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties
index 7474283..3e2f97a 100755
--- a/examples/test_case_data/sandbox-hdp21/kylin.properties
+++ b/examples/test_case_data/sandbox-hdp21/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index adc0dda..cd5d02c 100755
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/server/ServiceConfig.md
----------------------------------------------------------------------
diff --git a/server/ServiceConfig.md b/server/ServiceConfig.md
index 0f951be..fdeba33 100644
--- a/server/ServiceConfig.md
+++ b/server/ServiceConfig.md
@@ -20,8 +20,6 @@
##### kylin.query.cache.enabled
### Job
-##### kylin.job.jar
-##### kylin.job.log.dir
##### kylin.job.hdfs.working.dir
##### kylin.job.mapreduce.default.reduce.input.mb
##### kylin.job.mapreduce.default.reduce.count.ratio
@@ -35,6 +33,6 @@
##### kylin.job.concurrent.max.limit
##### kylin.job.command.exe.type
##### kylin.job.admin.dls
-##### kylin.job.step.timeout
+##### kylin.job.step.timeoutww
##### kylin.job.yarn.app.rest.check.interval.seconds
##### kylin.job.concurrent.max.limit
\ No newline at end of file
[45/50] incubator-kylin git commit: KYLIN-608 rename hllccounter
methond
Posted by li...@apache.org.
KYLIN-608 rename hllccounter methond
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ecfc3ccd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ecfc3ccd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ecfc3ccd
Branch: refs/heads/inverted-index
Commit: ecfc3ccd9a2c2a666363d78dced750060b8c6505
Parents: bde1e31
Author: honma <ho...@ebay.com>
Authored: Thu Feb 12 11:03:33 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Feb 12 11:03:33 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 72 ++------------------
.../common/util/HyperLogLogCounterTest.java | 4 +-
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../cardinality/ColumnCardinalityReducer.java | 4 +-
.../job/tools/ColumnCardinalityMapperTest.java | 4 +-
.../job/tools/ColumnCardinalityReducerTest.java | 2 +-
.../kylin/metadata/measure/HLLCSerializer.java | 4 +-
.../measure/fixedlen/FixedHLLCodec.java | 4 +-
.../query/sqlfunc/HLLDistinctCountAggFunc.java | 4 +-
9 files changed, 18 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 6ab0611..381dea6 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -169,63 +169,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
// ============================================================================
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
- // disable
- // due to
- // slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
- // lib
- // has
- // a
- // bug
- // at
- // the
- // moment
-
- public void writeCompactRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
+ public void writeRegisters(final ByteBuffer out) throws IOException {
final int indexLen = getRegisterIndexSize();
int size = size();
@@ -253,7 +197,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
}
- public void readCompactRegisters(ByteBuffer in) throws IOException {
+ public void readRegisters(ByteBuffer in) throws IOException {
byte scheme = in.get();
if (scheme == 0) { // map scheme
@@ -271,19 +215,11 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
}
- /**
- * For compressed output use writeCompactRegisters
- * @param out
- */
- public void writeRegisters(final ByteBuffer out) {
+ public void writeRegistersArray(final ByteBuffer out) {
out.put(this.registers);
}
- /**
- * For compressed input use readCompactRegisters
- * @param in
- */
- public void readRegisters(ByteBuffer in) {
+ public void readRegistersArray(ByteBuffer in) {
in.get(registers, 0, m);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index a7d275a..4a438fb 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -108,9 +108,9 @@ public class HyperLogLogCounterTest {
private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
long estimate = hllc.getCountEstimate();
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
- hllc.readCompactRegisters(buf);
+ hllc.readRegisters(buf);
Assert.assertEquals(estimate, hllc.getCountEstimate());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
index 329914a..dd7c720 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
@@ -94,7 +94,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWr
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit()));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
index b4b55d3..a9ac1a3 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
@@ -50,7 +50,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
for (BytesWritable v : values) {
ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
- hll.readCompactRegisters(buffer);
+ hll.readRegisters(buffer);
getHllc(skey).merge(hll);
hll.clear();
}
@@ -77,7 +77,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
// context.write(new Text("ErrorRate_" + key), new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
index 08c1611..e13289a 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
@@ -84,7 +84,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
+ hllc.readRegisters(ByteBuffer.wrap(bytes));
assertTrue(key1 > 0);
assertEquals(8, hllc.getCountEstimate());
}
@@ -117,7 +117,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
+ hllc.readRegisters(ByteBuffer.wrap(bytes));
System.out.println("ab\177ab".length());
assertTrue(key1 > 0);
assertEquals(1, hllc.getCountEstimate());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
index 6ad27a8..034c90e 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
@@ -67,7 +67,7 @@ public class ColumnCardinalityReducerTest {
}
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
return buf.array();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
index 20c6c21..e3fcbee 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
@@ -38,7 +38,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
@Override
public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
try {
- value.writeCompactRegisters(out);
+ value.writeRegisters(out);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -47,7 +47,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
@Override
public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
try {
- current.readCompactRegisters(in);
+ current.readRegisters(in);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
index d787cbc..c6d4dc9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -47,12 +47,12 @@ public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter>
@Override
public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- current.readRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
return current;
}
@Override
public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
- current.writeRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ current.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
index 8d9ace0..356c1e3 100644
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
+++ b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
@@ -118,12 +118,12 @@ public class HLLDistinctCountAggFunc {
}
@Override
- public void writeCompactRegisters(ByteBuffer out) throws IOException {
+ public void writeRegisters(ByteBuffer out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void readCompactRegisters(ByteBuffer in) throws IOException {
+ public void readRegisters(ByteBuffer in) throws IOException {
throw new UnsupportedOperationException();
}
[32/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/443b192c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/443b192c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/443b192c
Branch: refs/heads/inverted-index
Commit: 443b192ca4b19305761de5a47785f4b674b4c83c
Parents: dda5ee4 8466b53
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 15:47:26 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 15:47:26 2015 +0800
----------------------------------------------------------------------
.../kylin/common/hll/HyperLogLogPlusCounter.java | 16 +++-------------
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
jdbc/pom.xml | 5 -----
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
5 files changed, 4 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/285bd563
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/285bd563
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/285bd563
Branch: refs/heads/inverted-index
Commit: 285bd563fd2d7cb8e0bab0cad689450fa7860450
Parents: 59411d9 8245337
Author: jiazhong <ji...@ebay.com>
Authored: Wed Feb 11 19:18:56 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Wed Feb 11 19:18:56 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 5 +-
bin/stop-kylin.sh | 18 +++-
.../common/hll/HyperLogLogPlusCounter.java | 92 +------------------
.../64ac4f82-f2af-476e-85b9-f0805001014e.dict | Bin 554 -> 0 bytes
.../f5e85644-db92-42b5-9ad5-240ab227d7b0.dict | Bin 554 -> 0 bytes
.../16d8185c-ee6b-4f8c-a919-756d9809f937.dict | Bin 2515 -> 2630 bytes
.../c12ae49d-9dbe-4a58-b169-19afac317696.dict | Bin 2515 -> 2630 bytes
.../eaed91b0-4182-4ee5-a733-1047a622ee29.dict | Bin 2515 -> 2630 bytes
.../ec83ebce-7534-4e62-ac6d-7445ee141ab4.dict | Bin 2515 -> 2630 bytes
.../7228fad8-a764-4bd4-8934-50e0d7cbcb19.dict | Bin 2940 -> 3105 bytes
.../8f92faa4-7d2d-455c-8623-6e1d1b272afe.dict | Bin 2940 -> 3105 bytes
.../ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict | Bin 2940 -> 3105 bytes
.../eacccee7-d120-4f4d-97d0-c99a5b83ec32.dict | Bin 2940 -> 3105 bytes
.../2602386c-debb-4968-8d2f-b52b8215e385.dict | Bin 1841 -> 2000 bytes
.../4243889f-bc81-4807-a975-7041bbbf35e7.dict | Bin 1841 -> 2000 bytes
.../494683c1-52d1-4d9a-a3f7-8c53b30fd898.dict | Bin 1841 -> 2000 bytes
.../8b4b1c06-fb74-486b-a2ad-74420afebcda.dict | Bin 1841 -> 2000 bytes
.../ac520edd-f9d2-419f-a7de-587bfb97dc81.dict | Bin 1841 -> 2000 bytes
.../0410d2c4-4686-40bc-ba14-170042a2de94.dict | Bin 1498 -> 1537 bytes
.../51ff5e6c-22b3-444e-9915-d376a10f20cb.dict | Bin 1498 -> 1537 bytes
.../a4e57e55-48fc-4f25-a9c8-485deed25925.dict | Bin 1498 -> 1537 bytes
.../aceae914-4246-4251-a0c2-692fe7a300df.dict | Bin 1498 -> 1537 bytes
.../b298089f-9656-4693-b9b2-8fea46f06dd5.dict | Bin 1498 -> 1537 bytes
.../c2af25cf-6c79-45e6-a6f6-6d2a8ecc6592.dict | Bin 1498 -> 1537 bytes
.../0c5d77ec-316b-47e0-ba9a-0616be890ad6.dict | Bin 661 -> 640 bytes
.../14fe66b3-5956-498c-bd93-40182cac5510.dict | Bin 661 -> 640 bytes
.../1d383a36-81b9-4177-a822-04eab3683e5b.dict | Bin 661 -> 640 bytes
.../c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict | Bin 661 -> 640 bytes
.../0bec6bb3-1b0d-469c-8289-b8c4ca5d5001.dict | Bin 2189 -> 2264 bytes
.../31edf35b-ffca-4f24-8229-f87dc34e3087.dict | Bin 2189 -> 2264 bytes
.../652bd393-678a-4f16-a504-fd8ce1229355.dict | Bin 2189 -> 2264 bytes
.../792eb972-d046-48e6-9428-f6a3aed92fad.dict | Bin 2189 -> 2264 bytes
.../914536f8-1e14-43e9-b8ef-525bb5bdb3b2.dict | Bin 2189 -> 2264 bytes
.../d25e554e-deac-4e4a-9289-96f0d121d8fc.dict | Bin 2189 -> 2264 bytes
jdbc/pom.xml | 5 -
.../kylin/job/hadoop/hbase/TestHbaseClient.java | 2 +-
pom.xml | 15 ---
script/compress.sh | 8 +-
script/package.sh | 2 +-
script/prepare.sh | 4 +-
.../kylin/rest/service/ProjectService.java | 2 +-
41 files changed, 31 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/285bd563/bin/start-kylin.sh
----------------------------------------------------------------------
diff --cc bin/start-kylin.sh
index 735de53,b2544a5..b2544a5
mode 100644,100755..100644
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/285bd563/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --cc bin/stop-kylin.sh
index 806513b,c7a61b2..c7a61b2
mode 100644,100755..100644
--- a/bin/stop-kylin.sh
+++ b/bin/stop-kylin.sh
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/285bd563/script/package.sh
----------------------------------------------------------------------
diff --cc script/package.sh
index 018491e,556467a..556467a
mode 100644,100755..100644
--- a/script/package.sh
+++ b/script/package.sh
[02/50] incubator-kylin git commit: Merge branch 'inverted-index' of
https://github.com/KylinOLAP/Kylin into inverted-index
Posted by li...@apache.org.
Merge branch 'inverted-index' of https://github.com/KylinOLAP/Kylin into inverted-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e590e3e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e590e3e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e590e3e5
Branch: refs/heads/inverted-index
Commit: e590e3e5ac7b1871a8facaf6c007686c3e42d0e3
Parents: 8b1e74f 1f3bf76
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 12:30:22 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 12:30:22 2015 +0800
----------------------------------------------------------------------
common/pom.xml | 2 +-
.../common/persistence/HBaseConnection.java | 1 -
.../common/util/AbstractKylinTestCase.java | 80 -----
.../common/util/HBaseMetadataTestCase.java | 63 ----
.../common/util/HBaseMiniclusterHelper.java | 167 ----------
.../apache/kylin/common/util/HbaseImporter.java | 102 ------
.../common/util/LocalFileMetadataTestCase.java | 74 -----
.../common/util/AbstractKylinTestCase.java | 80 +++++
.../org/apache/kylin/common/util/BasicTest.java | 13 +-
.../common/util/HBaseMetadataTestCase.java | 63 ++++
.../common/util/HBaseMiniclusterHelper.java | 169 ++++++++++
.../apache/kylin/common/util/HbaseImporter.java | 104 ++++++
.../common/util/LocalFileMetadataTestCase.java | 74 +++++
cube/pom.xml | 10 +
dictionary/pom.xml | 9 +
invertedindex/pom.xml | 9 +
job/pom.xml | 9 +
.../org/apache/kylin/job/ExportHBaseData.java | 2 +-
metadata/pom.xml | 8 +
query/pom.xml | 7 +
.../apache/kylin/query/test/IIQueryTest.java | 9 +-
server/pom.xml | 8 +
storage/pom.xml | 8 +
.../endpoint/generated/IIProtos.java | 314 +++++++++----------
.../coprocessor/endpoint/protobuf/II.proto | 2 +-
25 files changed, 723 insertions(+), 664 deletions(-)
----------------------------------------------------------------------
[35/50] incubator-kylin git commit: comment format clean up
Posted by li...@apache.org.
comment format clean up
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/69959bea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/69959bea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/69959bea
Branch: refs/heads/inverted-index
Commit: 69959bea966c8f4e6ae7786025361332989b846e
Parents: 5edcf51
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:50:36 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:50:36 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/69959bea/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 3bcae86..467dfde 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -31,8 +31,8 @@ import com.google.common.hash.Hashing;
/**
* About compression, test on HLLC data shows
*
- * - LZF compression ratio is around 65%-80%, fast - GZIP compression ratio is
- * around 41%-46%, very slow
+ * - LZF compression ratio is around 65%-80%, fast
+ * - GZIP compression ratio is around 41%-46%, very slow
*
* @author yangli9
*/
[43/50] incubator-kylin git commit: KYLIN-608 support HLL at ii
storage
Posted by li...@apache.org.
KYLIN-608 support HLL at ii storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c5d329fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c5d329fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c5d329fe
Branch: refs/heads/inverted-index
Commit: c5d329fe098a0d4886f3e73a6ce0e99a621c8e67
Parents: 0a96d74
Author: honma <ho...@ebay.com>
Authored: Thu Feb 12 10:36:37 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Feb 12 10:36:37 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 14 +-
.../org/apache/kylin/common/util/BasicTest.java | 19 +-
.../common/util/HyperLogLogCounterTest.java | 12 +
.../invertedindex/index/RawTableRecord.java | 20 +-
.../apache/kylin/invertedindex/index/Slice.java | 330 +++++++++----------
.../kylin/invertedindex/index/TableRecord.java | 12 +-
.../measure/fixedlen/FixedHLLCodec.java | 14 +-
.../measure/fixedlen/FixedLenMeasureCodec.java | 3 +-
.../measure/fixedlen/FixedPointLongCodec.java | 6 +-
.../endpoint/EndpointAggregators.java | 117 ++++---
.../endpoint/EndpointTupleIterator.java | 18 +-
.../hbase/coprocessor/endpoint/IIEndpoint.java | 2 +-
.../org/apache/kylin/storage/tuple/Tuple.java | 12 +-
13 files changed, 325 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index c323b90..49a6756 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -28,10 +28,10 @@ import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.compress.utils.IOUtils;
+import org.apache.kylin.common.util.BytesUtil;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.BytesUtil;
import com.ning.compress.lzf.LZFDecoder;
import com.ning.compress.lzf.LZFEncoder;
@@ -72,8 +72,12 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
public void clear() {
- for (int i = 0; i < m; i++)
- registers[i] = 0;
+ byte zero = (byte) 0;
+ Arrays.fill(registers, zero);
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
}
public void add(String value) {
@@ -84,6 +88,10 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
add(hashFunc.hashBytes(value).asLong());
}
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
protected void add(long hash) {
int bucketMask = m - 1;
int bucket = (int) (hash & bucketMask);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 5952c33..0a33f9f 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -22,17 +22,9 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.params.HttpMethodParams;
import org.junit.Ignore;
import org.junit.Test;
-import org.slf4j.*;
-import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Created by honma on 10/17/14.
@@ -59,12 +51,19 @@ public class BasicTest {
System.out.printf("b");
}
+ private enum MetricType {
+ Count, DimensionAsMetric, DistinctCount, Normal
+ }
+
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
+ String x = MetricType.DimensionAsMetric.toString();
+ System.out.println(x);
+ MetricType y = MetricType.valueOf(x);
+ System.out.println(y == MetricType.DimensionAsMetric);
}
-
@Test
@Ignore("fix it later")
public void test2() throws IOException, ConfigurationException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index 088219f..a7d275a 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -204,6 +204,18 @@ public class HyperLogLogCounterTest {
System.out.println("Perf test result: " + duration / 1000 + " seconds");
}
+ @Test
+ public void testEquivalence() {
+ byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
+ byte[] b = new byte[] { 3, 4, 42 };
+ HyperLogLogPlusCounter ha = new HyperLogLogPlusCounter();
+ HyperLogLogPlusCounter hb = new HyperLogLogPlusCounter();
+ ha.add(a, 1, 3);
+ hb.add(b);
+
+ Assert.assertTrue(ha.getCountEstimate()==hb.getCountEstimate());
+ }
+
private HyperLogLogPlusCounter newHLLC() {
return new HyperLogLogPlusCounter(16);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
index 14ea62b..895fd4f 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
@@ -49,42 +49,44 @@ public class RawTableRecord implements Cloneable {
Arrays.fill(buf, Dictionary.NULL);
}
- protected boolean isMetric(int col) {
+ public boolean isMetric(int col) {
return digest.isMetrics(col);
}
- protected FixedLenMeasureCodec<LongWritable> codec(int col) {
+ public FixedLenMeasureCodec<LongWritable> codec(int col) {
return digest.codec(col);
}
- protected int length(int col) {
+ public int length(int col) {
return digest.length(col);
}
- protected int getColumnCount() {
+ public int getColumnCount() {
return digest.getColumnCount();
}
- protected void setValueID(int col, int id) {
+ public void setValueID(int col, int id) {
BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col));
}
- protected int getValueID(int col) {
+ public int getValueID(int col) {
return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col));
}
- protected void setValueMetrics(int col, LongWritable value) {
+ public void setValueMetrics(int col, LongWritable value) {
digest.codec(col).write(value, buf, digest.offset(col));
}
- protected LongWritable getValueMetrics(int col) {
- return digest.codec(col).read(buf, digest.offset(col));
+ public String getValueMetric(int col) {
+ digest.codec(col).read(buf, digest.offset(col));
+ return (String) digest.codec(col).getValue();
}
public byte[] getBytes() {
return buf;
}
+ //TODO is it possible to avoid copying?
public void setBytes(byte[] bytes, int offset, int length) {
assert buf.length == length;
System.arraycopy(bytes, offset, buf, 0, length);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
index fef9892..59dd9cd 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
@@ -31,171 +31,169 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
*/
public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
- TableRecordInfoDigest info;
- int nColumns;
-
- short shard;
- long timestamp;
- int nRecords;
- ColumnValueContainer[] containers;
-
- public Slice(TableRecordInfoDigest digest, short shard, long timestamp,
- ColumnValueContainer[] containers) {
- this.info = digest;
- this.nColumns = digest.getColumnCount();
-
- this.shard = shard;
- this.timestamp = timestamp;
- this.nRecords = containers[0].getSize();
- this.containers = containers;
-
- assert nColumns == containers.length;
- for (int i = 0; i < nColumns; i++) {
- assert nRecords == containers[i].getSize();
- }
- }
-
- public int getRecordCount() {
- return this.nRecords;
- }
-
- public short getShard() {
- return shard;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ColumnValueContainer[] getColumnValueContainers() {
- return containers;
- }
-
- public ColumnValueContainer getColumnValueContainer(int col) {
- return containers[col];
- }
-
- public Iterator<RawTableRecord> iterateWithBitmap(
- final ConciseSet resultBitMap) {
- if (resultBitMap == null) {
- return this.iterator();
- } else {
- return new Iterator<RawTableRecord>() {
- int i = 0;
- int iteratedCount = 0;
- int resultSize = resultBitMap.size();
-
- RawTableRecord rec = info.createTableRecordBytes();
- ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- @Override
- public boolean hasNext() {
- return iteratedCount < resultSize;
- }
-
- @Override
- public RawTableRecord next() {
- while (!resultBitMap.contains(i)) {
- i++;
- }
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- iteratedCount++;
- i++;
-
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
- }
-
- @Override
- public Iterator<RawTableRecord> iterator() {
- return new Iterator<RawTableRecord>() {
- int i = 0;
- RawTableRecord rec = info.createTableRecordBytes();
- ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- @Override
- public boolean hasNext() {
- return i < nRecords;
- }
-
- @Override
- public RawTableRecord next() {
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- i++;
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((info == null) ? 0 : info.hashCode());
- result = prime * result + shard;
- result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Slice other = (Slice) obj;
- if (info == null) {
- if (other.info != null)
- return false;
- } else if (!info.equals(other.info))
- return false;
- if (shard != other.shard)
- return false;
- if (timestamp != other.timestamp)
- return false;
- return true;
- }
-
- @Override
- public int compareTo(Slice o) {
- int comp = this.shard - o.shard;
- if (comp != 0)
- return comp;
-
- comp = (int) (this.timestamp - o.timestamp);
- return comp;
- }
+ TableRecordInfoDigest info;
+ int nColumns;
+
+ short shard;
+ long timestamp;
+ int nRecords;
+ ColumnValueContainer[] containers;
+
+ public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
+ this.info = digest;
+ this.nColumns = digest.getColumnCount();
+
+ this.shard = shard;
+ this.timestamp = timestamp;
+ this.nRecords = containers[0].getSize();
+ this.containers = containers;
+
+ assert nColumns == containers.length;
+ for (int i = 0; i < nColumns; i++) {
+ assert nRecords == containers[i].getSize();
+ }
+ }
+
+ public int getRecordCount() {
+ return this.nRecords;
+ }
+
+ public short getShard() {
+ return shard;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public ColumnValueContainer[] getColumnValueContainers() {
+ return containers;
+ }
+
+ public ColumnValueContainer getColumnValueContainer(int col) {
+ return containers[col];
+ }
+
+ public Iterator<RawTableRecord> iterateWithBitmap(final ConciseSet resultBitMap) {
+ if (resultBitMap == null) {
+ return this.iterator();
+ } else {
+ final RawTableRecord rec = info.createTableRecordBytes();
+ final ImmutableBytesWritable temp = new ImmutableBytesWritable();
+
+ return new Iterator<RawTableRecord>() {
+ int i = 0;
+ int iteratedCount = 0;
+ int resultSize = resultBitMap.size();
+
+ @Override
+ public boolean hasNext() {
+ return iteratedCount < resultSize;
+ }
+
+ @Override
+ public RawTableRecord next() {
+ while (!resultBitMap.contains(i)) {
+ i++;
+ }
+ for (int col = 0; col < nColumns; col++) {
+ containers[col].getValueAt(i, temp);
+ rec.setValueBytes(col, temp);
+ }
+ iteratedCount++;
+ i++;
+
+ return rec;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+ }
+
+ @Override
+ public Iterator<RawTableRecord> iterator() {
+ return new Iterator<RawTableRecord>() {
+ int i = 0;
+ RawTableRecord rec = info.createTableRecordBytes();
+ ImmutableBytesWritable temp = new ImmutableBytesWritable();
+
+ @Override
+ public boolean hasNext() {
+ return i < nRecords;
+ }
+
+ @Override
+ public RawTableRecord next() {
+ for (int col = 0; col < nColumns; col++) {
+ containers[col].getValueAt(i, temp);
+ rec.setValueBytes(col, temp);
+ }
+ i++;
+ return rec;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#hashCode()
+ */
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((info == null) ? 0 : info.hashCode());
+ result = prime * result + shard;
+ result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+ return result;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Slice other = (Slice) obj;
+ if (info == null) {
+ if (other.info != null)
+ return false;
+ } else if (!info.equals(other.info))
+ return false;
+ if (shard != other.shard)
+ return false;
+ if (timestamp != other.timestamp)
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(Slice o) {
+ int comp = this.shard - o.shard;
+ if (comp != 0)
+ return comp;
+
+ comp = (int) (this.timestamp - o.timestamp);
+ return comp;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
index 1abbe18..3b8d969 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
@@ -73,7 +73,7 @@ public class TableRecord implements Cloneable {
return rawRecord.length(col);
}
- public List<String> getValueList() {
+ public List<String> getOriginTableColumnValues() {
List<String> ret = Lists.newArrayList();
for (int i = 0; i < info.nColumns; ++i) {
ret.add(getValueString(i));
@@ -91,9 +91,13 @@ public class TableRecord implements Cloneable {
}
}
+ /**
+ * get value of columns which belongs to the original table columns.
+ * i.e. columns like min_xx, max_yy will never appear
+ */
public String getValueString(int col) {
if (rawRecord.isMetric(col))
- return rawRecord.codec(col).toString(getValueMetrics(col));
+ return getValueMetric(col);
else
return info.dict(col).getValueFromId(rawRecord.getValueID(col));
}
@@ -106,8 +110,8 @@ public class TableRecord implements Cloneable {
rawRecord.setValueMetrics(col, value);
}
- private LongWritable getValueMetrics(int col) {
- return rawRecord.getValueMetrics(col);
+ private String getValueMetric(int col) {
+ return rawRecord.getValueMetric(col);
}
public short getShard() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
index 9f6a1ba..d787cbc 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -1,11 +1,10 @@
package org.apache.kylin.metadata.measure.fixedlen;
+import java.nio.ByteBuffer;
+
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.measure.HLLCSerializer;
import org.apache.kylin.metadata.model.DataType;
-import java.util.Map;
-
/**
* Created by Hongbin Ma(Binmahone) on 2/10/15.
*/
@@ -42,17 +41,18 @@ public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter>
}
@Override
- public String toString(HyperLogLogPlusCounter value) {
- return String.valueOf(value.getCountEstimate());
+ public Object getValue() {
+ return current;
}
@Override
public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- return serializer.deserialize();
+ current.readRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ return current;
}
@Override
public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
-
+ current.writeRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
index 41a6356..650432a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
@@ -33,7 +33,8 @@ abstract public class FixedLenMeasureCodec<T> {
abstract public T valueOf(String value);
- abstract public String toString(T value);
+
+ abstract public Object getValue();
abstract public T read(byte[] buf, int offset);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
index f27e446..9ccb479 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedPointLongCodec.java
@@ -61,11 +61,11 @@ public class FixedPointLongCodec extends FixedLenMeasureCodec<LongWritable> {
}
@Override
- public String toString(LongWritable value) {
+ public String getValue() {
if (scale == 0)
- return value.toString();
+ return current.toString();
else
- return "" + (new BigDecimal(value.get()).divide(scalePower));
+ return "" + (new BigDecimal(current.get()).divide(scalePower));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
index b199862..c6d8c49 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
@@ -18,23 +18,24 @@
package org.apache.kylin.storage.hbase.coprocessor.endpoint;
-import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
+import java.util.List;
-import com.yammer.metrics.core.Metric;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.invertedindex.index.RawTableRecord;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
-import org.apache.hadoop.io.LongWritable;
-import java.nio.ByteBuffer;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* @author honma
@@ -49,6 +50,13 @@ public class EndpointAggregators {
private static class MetricInfo {
private MetricType type;
private int refIndex = -1;
+ private int presision = -1;
+
+ public MetricInfo(MetricType type, int refIndex, int presision) {
+ this.type = type;
+ this.refIndex = refIndex;
+ this.presision = presision;
+ }
public MetricInfo(MetricType type, int refIndex) {
this.type = type;
@@ -58,6 +66,7 @@ public class EndpointAggregators {
public MetricInfo(MetricType type) {
this.type = type;
}
+
}
public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) {
@@ -83,7 +92,7 @@ public class EndpointAggregators {
}
if (functionDesc.isCountDistinct()) {
- metricInfos[i] = new MetricInfo(MetricType.DistinctCount, index);
+ metricInfos[i] = new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision());
} else {
metricInfos[i] = new MetricInfo(MetricType.Normal, index);
}
@@ -96,8 +105,11 @@ public class EndpointAggregators {
final String[] funcNames;
final String[] dataTypes;
final MetricInfo[] metricInfos;
- final TableRecordInfoDigest tableRecordInfo;
+ final transient TableRecordInfoDigest tableRecordInfoDigest;
+ final transient RawTableRecord rawTableRecord;
+ final transient ImmutableBytesWritable byteBuffer;
+ final transient HyperLogLogPlusCounter[] hllcs;
final transient FixedLenMeasureCodec[] measureSerializers;
final transient Object[] metricValues;
@@ -107,8 +119,11 @@ public class EndpointAggregators {
this.funcNames = funcNames;
this.dataTypes = dataTypes;
this.metricInfos = metricInfos;
- this.tableRecordInfo = tableInfo;
+ this.tableRecordInfoDigest = tableInfo;
+ this.rawTableRecord = tableInfo.createTableRecordBytes();
+ this.byteBuffer = new ImmutableBytesWritable();
+ this.hllcs = new HyperLogLogPlusCounter[this.metricInfos.length];
this.metricValues = new Object[funcNames.length];
this.measureSerializers = new FixedLenMeasureCodec[funcNames.length];
for (int i = 0; i < this.measureSerializers.length; ++i) {
@@ -116,8 +131,8 @@ public class EndpointAggregators {
}
}
- public TableRecordInfoDigest getTableRecordInfo() {
- return tableRecordInfo;
+ public TableRecordInfoDigest getTableRecordInfoDigest() {
+ return tableRecordInfoDigest;
}
public boolean isEmpty() {
@@ -133,35 +148,41 @@ public class EndpointAggregators {
return aggrs;
}
+ /**
+ * this method is heavily called at coprocessor side,
+ * Make sure as little object creation as possible
+ */
public void aggregate(MeasureAggregator[] measureAggrs, byte[] row) {
- int rawIndex = 0;
- int columnCount = tableRecordInfo.getColumnCount();
-
- for (int columnIndex = 0; columnIndex < columnCount; ++columnIndex) {
- for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
- if (metricInfos[metricIndex].refIndex == columnIndex) {
- if (metricInfos[metricIndex].type == MetricType.Normal) {
- //normal column values to aggregate
- measureAggrs[metricIndex].aggregate(measureSerializers[metricIndex].read(row, rawIndex));
- } else if (metricInfos[metricIndex].type == MetricType.DistinctCount) {
- if (tableRecordInfo.isMetrics(columnCount)) {
- measureAggrs[metricIndex].aggregate(measureSerializers[metricIndex].read(row, rawIndex));
- } else {
- //TODO: for unified dictionary, this is okay. but if different data blocks uses different dictionary, we'll have to aggregate original data
- measureAggrs[metricIndex].aggregate(tableRecordInfo.);
- }
- }
+
+ rawTableRecord.setBytes(row, 0, row.length);
+
+ for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
+
+ MetricInfo metricInfo = metricInfos[metricIndex];
+ MeasureAggregator aggregator = measureAggrs[metricIndex];
+ FixedLenMeasureCodec measureSerializer = measureSerializers[metricIndex];
+
+ //get the raw bytes
+ rawTableRecord.getValueBytes(metricInfo.refIndex, byteBuffer);
+
+ if (metricInfo.type == MetricType.Normal) {
+ aggregator.aggregate(measureSerializer.read(byteBuffer.get(), byteBuffer.getOffset()));
+ } else if (metricInfo.type == MetricType.DistinctCount) {
+ //TODO: for unified dictionary, this is okay. but if different data blocks uses different dictionary, we'll have to aggregate original data
+ HyperLogLogPlusCounter hllc = hllcs[metricIndex];
+ if (hllc == null) {
+ hllc = new HyperLogLogPlusCounter(metricInfo.presision);
}
+ hllc.clear();
+ hllc.add(byteBuffer.get(), byteBuffer.getOffset(), byteBuffer.getLength());
+ aggregator.aggregate(hllc);
}
- rawIndex += tableRecordInfo.length(columnIndex);
}
//aggregate for "count"
for (int i = 0; i < metricInfos.length; ++i) {
if (metricInfos[i].type == MetricType.Count) {
measureAggrs[i].aggregate(ONE);
- } else if (metricInfos[i].type == MetricType.DistinctCount) {
-
}
}
}
@@ -184,11 +205,12 @@ public class EndpointAggregators {
return metricBytesOffset;
}
- public List<String> deserializeMetricValues(byte[] metricBytes, int offset) {
- List<String> ret = Lists.newArrayList();
+ public List<Object> deserializeMetricValues(byte[] metricBytes, int offset) {
+ List<Object> ret = Lists.newArrayList();
int metricBytesOffset = offset;
for (int i = 0; i < measureSerializers.length; i++) {
- String valueString = measureSerializers[i].toString(measureSerializers[i].read(metricBytes, metricBytesOffset));
+ measureSerializers[i].read(metricBytes, metricBytesOffset);
+ Object valueString = measureSerializers[i].getValue();
metricBytesOffset += measureSerializers[i].getLength();
ret.add(valueString);
}
@@ -215,18 +237,37 @@ public class EndpointAggregators {
public void serialize(EndpointAggregators value, ByteBuffer out) {
BytesUtil.writeAsciiStringArray(value.funcNames, out);
BytesUtil.writeAsciiStringArray(value.dataTypes, out);
- BytesUtil.writeIntArray(value.metricInfos, out);
- BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfo), out);
+
+ BytesUtil.writeVInt(value.metricInfos.length, out);
+ for (int i = 0; i < value.metricInfos.length; ++i) {
+ MetricInfo metricInfo = value.metricInfos[i];
+ BytesUtil.writeAsciiString(metricInfo.type.toString(), out);
+ BytesUtil.writeVInt(metricInfo.refIndex, out);
+ BytesUtil.writeVInt(metricInfo.presision, out);
+ }
+
+ BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfoDigest), out);
}
@Override
public EndpointAggregators deserialize(ByteBuffer in) {
+
String[] funcNames = BytesUtil.readAsciiStringArray(in);
String[] dataTypes = BytesUtil.readAsciiStringArray(in);
- int[] refColIndex = BytesUtil.readIntArray(in);
+
+ int metricInfoLength = BytesUtil.readVInt(in);
+ MetricInfo[] infos = new MetricInfo[metricInfoLength];
+ for (int i = 0; i < infos.length; ++i) {
+ MetricType type = MetricType.valueOf(BytesUtil.readAsciiString(in));
+ int refIndex = BytesUtil.readVInt(in);
+ int presision = BytesUtil.readVInt(in);
+ infos[i] = new MetricInfo(type, refIndex, presision);
+ }
+
byte[] temp = BytesUtil.readByteArray(in);
TableRecordInfoDigest tableInfo = TableRecordInfoDigest.deserialize(temp);
- return new EndpointAggregators(funcNames, dataTypes, refColIndex, tableInfo);
+
+ return new EndpointAggregators(funcNames, dataTypes, infos, tableInfo);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index d63bc0d..465f7f3 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -266,7 +266,7 @@ public class EndpointTupleIterator implements ITupleIterator {
//not thread safe!
private TableRecord tableRecord;
- private List<String> measureValues;
+ private List<Object> measureValues;
private Tuple tuple;
public SingleRegionTupleIterator(List<IIProtos.IIResponse.IIRow> rows) {
@@ -305,26 +305,32 @@ public class EndpointTupleIterator implements ITupleIterator {
}
- private ITuple makeTuple(TableRecord tableRecord, List<String> measureValues) {
+ private ITuple makeTuple(TableRecord tableRecord, List<Object> measureValues) {
// groups
- List<String> columnValues = tableRecord.getValueList();
+ List<String> columnValues = tableRecord.getOriginTableColumnValues();
for (int i = 0; i < columnNames.size(); i++) {
TblColRef column = columns.get(i);
if (!tuple.hasColumn(column)) {
continue;
}
- tuple.setValue(columnNames.get(i), columnValues.get(i));
+ tuple.setDimensionValue(columnNames.get(i), columnValues.get(i));
}
if (measureValues != null) {
for (int i = 0; i < measures.size(); ++i) {
if (!measures.get(i).isAppliedOnDimension()) {
- tuple.setValue(measures.get(i).getRewriteFieldName(), measureValues.get(i));
+ String fieldName = measures.get(i).getRewriteFieldName();
+ Object value = measureValues.get(i);
+ String dataType = tuple.getDataType(fieldName);
+ //TODO: currently in II all metrics except HLLC is returned as String
+ if (dataType.toLowerCase().equalsIgnoreCase("hllc")) {
+ value = Tuple.convertOptiqCellValue((String) value, dataType);
+ }
+ tuple.setMeasureValue(fieldName, value);
}
}
}
return tuple;
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
index db68803..4852e3b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/IIEndpoint.java
@@ -85,7 +85,7 @@ public class IIEndpoint extends IIProtos.RowsService implements Coprocessor, Cop
aggregators = EndpointAggregators.deserialize(request.getAggregator().toByteArray());
filter = CoprocessorFilter.deserialize(request.getFilter().toByteArray());
- TableRecordInfoDigest tableRecordInfoDigest = aggregators.getTableRecordInfo();
+ TableRecordInfoDigest tableRecordInfoDigest = aggregators.getTableRecordInfoDigest();
IIProtos.IIResponse response = null;
RegionScanner innerScanner = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c5d329fe/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
index 7b5fe1f..dd19e0c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
+++ b/storage/src/main/java/org/apache/kylin/storage/tuple/Tuple.java
@@ -78,18 +78,17 @@ public class Tuple implements ITuple {
return values[index];
}
+ public String getDataType(String fieldName) {
+ return info.getDataType(fieldName);
+ }
+
private void setFieldObjectValue(String fieldName, Object fieldValue) {
int index = info.getFieldIndex(fieldName);
values[index] = fieldValue;
}
- public void setValue(String fieldName, String fieldValue) {
- this.setDimensionValue(fieldName, fieldValue);
- }
-
public void setDimensionValue(String fieldName, String fieldValue) {
- String dataType = info.getDataType(fieldName);
- Object objectValue = convertOptiqCellValue(fieldValue, dataType);
+ Object objectValue = convertOptiqCellValue(fieldValue, getDataType(fieldName));
setFieldObjectValue(fieldName, objectValue);
}
@@ -121,6 +120,7 @@ public class Tuple implements ITuple {
return sb.toString();
}
+
public static Object convertOptiqCellValue(String strValue, String dataType) {
if (strValue == null)
return null;
[24/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/5712b1d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/5712b1d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/5712b1d0
Branch: refs/heads/inverted-index
Commit: 5712b1d0f4930d5df4594f9d2e1b6e7135b5b5b7
Parents: 78689e3
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 11:50:21 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 11:50:21 2015 +0800
----------------------------------------------------------------------
script/compress.sh | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/5712b1d0/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index 01a8376..20b4257 100644
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -13,6 +13,7 @@ fi
#package tgz
echo 'package tgz'
+rm -rf kylin-${version}
mkdir kylin-${version}
cp -r lib bin conf tomcat kylin-${version}
tar -cvzf kylin-${version}.tgz kylin-${version}
[33/50] incubator-kylin git commit: drop compression feature inside
HLLC SerDes
Posted by li...@apache.org.
drop compression feature inside HLLC SerDes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d84c1f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d84c1f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d84c1f1
Branch: refs/heads/inverted-index
Commit: 8d84c1f11dd5809ab01624c3c9c1a96346d8e78e
Parents: 8466b53
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:49:35 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:49:35 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 76 +-------------------
1 file changed, 1 insertion(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d84c1f1/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 18b3af1..3bcae86 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -18,22 +18,15 @@
package org.apache.kylin.common.hll;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.Arrays;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
+import org.apache.kylin.common.util.BytesUtil;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.BytesUtil;
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
/**
* About compression, test on HLLC data shows
@@ -168,54 +161,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
// ============================================================================
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes, disable due to slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF lib has a bug at the moment
-
public void writeRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
-
final int indexLen = getRegisterIndexSize();
int size = size();
@@ -240,30 +186,10 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
out.put(registers[i]);
}
}
-
- // do compression if needed
- int len = out.position() - startPos;
- if (len < COMPRESSION_THRESHOLD)
- return;
-
- scheme |= COMPRESSION_FLAG;
- byte[] compressed = DEFAULT_COMPRESSOR.compress(out, startPos + 1, len - 1);
- out.position(startPos);
- out.put(scheme);
- BytesUtil.writeVInt(compressed.length, out);
- out.put(compressed);
}
public void readRegisters(ByteBuffer in) throws IOException {
byte scheme = in.get();
- if ((scheme & COMPRESSION_FLAG) > 0) {
- scheme ^= COMPRESSION_FLAG;
- int compressedLen = BytesUtil.readVInt(in);
- int end = in.position() + compressedLen;
- byte[] decompressed = DEFAULT_COMPRESSOR.decompress(in, in.position(), compressedLen);
- in.position(end);
- in = ByteBuffer.wrap(decompressed);
- }
if (scheme == 0) { // map scheme
clear();
[20/50] incubator-kylin git commit: refacotr jobList&cubeList page
Posted by li...@apache.org.
refacotr jobList&cubeList page
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a2383a54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a2383a54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a2383a54
Branch: refs/heads/inverted-index
Commit: a2383a5459362c8f152cd2cdff643231df5f8459
Parents: 09f4bc3
Author: jiazhong <ji...@ebay.com>
Authored: Tue Feb 10 20:47:14 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Feb 10 20:47:14 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 51 +++++++-----------------
webapp/app/js/controllers/job.js | 41 +++++++------------
webapp/app/js/model/cubeListModel.js | 65 +++++++++++++++++++++++++++++++
webapp/app/js/model/jobListModel.js | 61 +++++++++++++++++++++++++++++
webapp/app/partials/cubes/cubes.html | 14 +++----
webapp/app/partials/jobs/jobs.html | 14 +++----
6 files changed, 169 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
old mode 100755
new mode 100644
index 31c4557..d4fd0a5
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -19,9 +19,10 @@
'use strict';
KylinApp
- .controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService,SweetAlert,loadingRequest,$log,cubeConfig,ProjectModel,ModelService,MetaModel) {
+ .controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService,SweetAlert,loadingRequest,$log,cubeConfig,ProjectModel,ModelService,MetaModel,CubeList) {
$scope.cubeConfig = cubeConfig;
+ $scope.cubeList = CubeList;
$scope.listParams={
cubeName: $routeParams.cubeName,
@@ -30,7 +31,7 @@ KylinApp
if($routeParams.projectName){
$scope.projectModel.setSelectedProject($routeParams.projectName);
}
- $scope.cubes = [];
+ CubeList.removeAll();
$scope.loading = false;
$scope.action = {};
@@ -45,7 +46,6 @@ KylinApp
}
offset = (!!offset) ? offset : 0;
limit = (!!limit) ? limit : 20;
- var defer = $q.defer();
var queryParam = {offset: offset, limit: limit};
if ($scope.listParams.cubeName) {
@@ -54,40 +54,18 @@ KylinApp
queryParam.projectName = $scope.projectModel.selectedProject;
$scope.loading = true;
- CubeService.list(queryParam, function (cubes) {
- angular.forEach(cubes, function (cube, index) {
- if(cube.name){
- $scope.listAccess(cube, 'CubeInstance');
- if (cube.segments && cube.segments.length > 0) {
- for(var i= cube.segments.length-1;i>=0;i--){
- if(cube.segments[i].status==="READY"){
- cube.last_build_time = cube.segments[i].last_build_time;
- break;
- }else if(i===0){
- cube.last_build_time = cube.create_time_utc;
- }
- }
- } else {
- cube.last_build_time = cube.create_time_utc;
- }
- if($routeParams.showDetail == 'true'){
- cube.showDetail = true;
- $scope.loadDetail(cube);
- }
- }
- });
- cubes = _.filter(cubes,function(cube){return cube.name!=undefined});
- $scope.cubes = $scope.cubes.concat(cubes);
+
+ var defer = $q.defer();
+ return CubeList.list(queryParam).then(function(resp){
$scope.loading = false;
- defer.resolve(cubes.length);
+ defer.resolve(resp);
+ defer.promise;
});
-
- return defer.promise;
};
$scope.$watch('projectModel.selectedProject', function (newValue, oldValue) {
if(newValue!=oldValue||newValue==null){
- $scope.cubes=[];
+ CubeList.removeAll();
$scope.reload();
}
@@ -179,7 +157,7 @@ KylinApp
CubeService.purge({cubeId: cube.name}, {}, function (result) {
loadingRequest.hide();
- $scope.cubes=[];
+ CubeList.removeAll();
$scope.reload();
SweetAlert.swal('Success!', 'Purge job was submitted successfully', 'success');
},function(e){
@@ -248,10 +226,11 @@ KylinApp
CubeService.drop({cubeId: cube.name}, {}, function (result) {
loadingRequest.hide();
- var cubeIndex = $scope.cubes.indexOf(cube);
- if (cubeIndex > -1) {
- $scope.cubes.splice(cubeIndex, 1);
- }
+// var cubeIndex = CubeList.cubes.indexOf(cube);
+// if (cubeIndex > -1) {
+// $scope.cubes.splice(cubeIndex, 1);
+// }
+ CubeList.removeCube(cube);
SweetAlert.swal('Success!', 'Cube drop is done successfully', 'success');
},function(e){
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/js/controllers/job.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js
old mode 100755
new mode 100644
index feeb296..c5fd311
--- a/webapp/app/js/controllers/job.js
+++ b/webapp/app/js/controllers/job.js
@@ -19,10 +19,11 @@
'use strict';
KylinApp
- .controller('JobCtrl', function ($scope, $q, $routeParams, $interval, $modal, ProjectService, MessageService, JobService,SweetAlert,loadingRequest,UserService,jobConfig) {
+ .controller('JobCtrl', function ($scope, $q, $routeParams, $interval, $modal, ProjectService, MessageService, JobService,SweetAlert,loadingRequest,UserService,jobConfig,JobList) {
+
+ $scope.jobList = JobList;
$scope.jobConfig = jobConfig;
$scope.cubeName = null;
- $scope.jobs = {};
$scope.projects = [];
$scope.action = {};
@@ -71,27 +72,13 @@ KylinApp
limit: limit
};
$scope.state.loading = true;
- JobService.list(jobRequest, function (jobs) {
- angular.forEach(jobs, function (job) {
- var id = job.uuid;
- if (angular.isDefined($scope.jobs[id])) {
- if (job.last_modified != $scope.jobs[id].last_modified) {
- $scope.jobs[id] = job;
- } else {
- }
- } else {
- $scope.jobs[id] = job;
- }
- });
+ var defer = $q.defer();
+ return JobList.list(jobRequest).then(function(resp){
$scope.state.loading = false;
- if (angular.isDefined($scope.state.selectedJob)) {
- $scope.state.selectedJob = $scope.jobs[selectedJob.uuid];
- }
- defer.resolve(jobs.length);
+ defer.resolve(resp);
+ defer.promise;
});
-
- return defer.promise;
}
$scope.reload = function () {
@@ -102,7 +89,7 @@ KylinApp
$scope.$watch('projectModel.selectedProject', function (newValue, oldValue) {
if(newValue!=oldValue||newValue==null){
- $scope.jobs={};
+ JobList.removeAll();
$scope.state.projectName = newValue;
$scope.reload();
}
@@ -121,9 +108,9 @@ KylinApp
loadingRequest.show();
JobService.resume({jobId: job.uuid}, {}, function (job) {
loadingRequest.hide();
- $scope.jobs[job.uuid] = job;
+ JobList.jobs[job.uuid] = job;
if (angular.isDefined($scope.state.selectedJob)) {
- $scope.state.selectedJob = $scope.jobs[ $scope.state.selectedJob.uuid];
+ $scope.state.selectedJob = JobList.jobs[ $scope.state.selectedJob.uuid];
}
SweetAlert.swal('Success!', 'Job has been resumed successfully!', 'success');
},function(e){
@@ -154,9 +141,9 @@ KylinApp
JobService.cancel({jobId: job.uuid}, {}, function (job) {
loadingRequest.hide();
$scope.safeApply(function() {
- $scope.jobs[job.uuid] = job;
+ JobList.jobs[job.uuid] = job;
if (angular.isDefined($scope.state.selectedJob)) {
- $scope.state.selectedJob = $scope.jobs[ $scope.state.selectedJob.uuid];
+ $scope.state.selectedJob = JobList.jobs[ $scope.state.selectedJob.uuid];
}
});
@@ -181,8 +168,8 @@ KylinApp
internalOpenModal();
var stepId = $scope.state.selectedStep.sequence_id;
JobService.stepOutput({jobId: $scope.state.selectedJob.uuid, propValue: $scope.state.selectedStep.id}, function (result) {
- if (angular.isDefined($scope.jobs[result['jobId']])) {
- var tjob = $scope.jobs[result['jobId']];
+ if (angular.isDefined(JobList.jobs[result['jobId']])) {
+ var tjob = JobList.jobs[result['jobId']];
tjob.steps[stepId].cmd_output = result['cmd_output'];
tjob.steps[stepId].loadingOp = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/js/model/cubeListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeListModel.js b/webapp/app/js/model/cubeListModel.js
new file mode 100644
index 0000000..3cd1b2b
--- /dev/null
+++ b/webapp/app/js/model/cubeListModel.js
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+KylinApp.service('CubeList',function(CubeService,$q){
+ var cubes=[];
+ var _this = this;
+
+ this.list = function(queryParam){
+
+ var defer = $q.defer();
+ CubeService.list(queryParam, function (_cubes) {
+ angular.forEach(_cubes, function (cube, index) {
+ if(cube.name){
+// $scope.listAccess(cube, 'CubeInstance');
+ if (cube.segments && cube.segments.length > 0) {
+ for(var i= cube.segments.length-1;i>=0;i--){
+ if(cube.segments[i].status==="READY"){
+ cube.last_build_time = cube.segments[i].last_build_time;
+ break;
+ }else if(i===0){
+ cube.last_build_time = cube.create_time_utc;
+ }
+ }
+ } else {
+ cube.last_build_time = cube.create_time_utc;
+ }
+ }
+ });
+ _cubes = _.filter(_cubes,function(cube){return cube.name!=undefined});
+ _this.cubes = _this.cubes.concat(_cubes);
+ defer.resolve(_this.cubes.length);
+ },function(){
+ defer.reject("Failed to load cubes");
+ });
+ return defer.promise;
+
+ };
+
+ this.removeCube = function(cube){
+ var cubeIndex = _this.cubes.indexOf(cube);
+ if (cubeIndex > -1) {
+ _this.cubes.splice(cubeIndex, 1);
+ }
+ }
+
+ this.removeAll = function(){
+ _this.cubes=[];
+ };
+
+});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/js/model/jobListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/jobListModel.js b/webapp/app/js/model/jobListModel.js
new file mode 100644
index 0000000..53f5e9a
--- /dev/null
+++ b/webapp/app/js/model/jobListModel.js
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+KylinApp.service('JobList',function(JobService,$q){
+ var jobs=[];
+ var _this = this;
+
+ this.list = function(jobRequest){
+
+ var defer = $q.defer();
+ JobService.list(jobRequest, function (jobs) {
+ angular.forEach(jobs, function (job) {
+ var id = job.uuid;
+ if (angular.isDefined(_this.jobs[id])) {
+ if (job.last_modified != _this.jobs[id].last_modified) {
+ _this.jobs[id] = job;
+ } else {
+ }
+ } else {
+ _this.jobs[id] = job;
+ }
+ });
+
+// $scope.state.loading = false;
+// if (angular.isDefined($scope.state.selectedJob)) {
+// $scope.state.selectedJob = $scope.jobs[selectedJob.uuid];
+// }
+ defer.resolve(jobs.length);
+ });
+
+ return defer.promise;
+
+ };
+
+// this.removeCube = function(cube){
+// var cubeIndex = _this.cubes.indexOf(cube);
+// if (cubeIndex > -1) {
+// _this.cubes.splice(cubeIndex, 1);
+// }
+// }
+
+ this.removeAll = function(){
+ _this.jobs=[];
+ };
+
+});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
old mode 100755
new mode 100644
index 8b69fdf..19fd7d1
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -39,10 +39,10 @@
</div>
<!--Cube Name-->
<div class="col-xs-3">
- <form ng-submit="cubes=[];list()" style="display: inline" >
+ <form ng-submit="cubeList.removeAll();list()" style="display: inline" >
<span class="input-icon input-icon-right nav-search"><b>Cube Name:</b>
<input type="text" placeholder="Filter ..." class="nav-search-input" ng-model="listParams.cubeName" />
- <i class="ace-icon fa fa-search blue" ng-click="cubes=[];list()"></i>
+ <i class="ace-icon fa fa-search blue" ng-click="cubeList.removeAll();list()"></i>
</span>
</form>
</div>
@@ -52,13 +52,13 @@
</div>
</div>
-<div ng-if="!loading && cubes.length == 0">
+<div ng-if="!loading && cubeList.cubes.length == 0">
<div no-result text="No Cube."></div>
</div>
<loading ng-if="loading" text="Loading Cubes..."></loading>
<!--Table-->
-<div ng-if="cubes.length > 0" class="dataTables_wrapper no-footer">
+<div ng-if="cubeList.cubes.length > 0" class="dataTables_wrapper no-footer">
<div class="row">
<div class="col-xs-12"><label class="table-header-text">Cubes</label></div>
</div>
@@ -81,7 +81,7 @@
</tr>
</thead>
<!--Body-->
- <tbody ng-repeat="cube in cubes | orderObjectBy:state.filterAttr:state.filterReverse">
+ <tbody ng-repeat="cube in cubeList.cubes | orderObjectBy:state.filterAttr:state.filterReverse">
<tr ng-class="{accordion:true}" style="cursor: pointer" ng-click="cube.showDetail=!cube.showDetail;loadDetail(cube)">
<td>
<i ng-show="!cube.showDetail" class="fa fa-chevron-circle-right blue"></i>
@@ -151,11 +151,11 @@
<div class="row">
<div class="col-xs-12">
- <kylin-pagination data="cubes" load-func="list" action="action"/>
+ <kylin-pagination data="cubeList.cubes" load-func="list" action="action"/>
</div>
</div>
<div class="pull-left font-color-default" style="font-size: 15px" >
- <strong>Storage: {{getTotalSize(cubes)}}</strong>
+ <strong>Storage: {{getTotalSize(cubeList.cubes)}}</strong>
</div>
<div ng-include="'partials/jobs/job_submit.html'"></div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a2383a54/webapp/app/partials/jobs/jobs.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/jobs/jobs.html b/webapp/app/partials/jobs/jobs.html
old mode 100755
new mode 100644
index b35255c..aba9ce6
--- a/webapp/app/partials/jobs/jobs.html
+++ b/webapp/app/partials/jobs/jobs.html
@@ -39,10 +39,10 @@
</div>
<!--Cube Name: -->
<div class="col-xs-3">
- <form ng-submit="jobs={};reload()" style="display: inline" >
+ <form ng-submit="jobList.removeAll();reload()" style="display: inline" >
<span class="input-icon input-icon-right nav-search"><b>Cube Name:</b>
<input type="text" placeholder="Filter ..." class="nav-search-input" ng-model="cubeName" />
- <i class="ace-icon fa fa-search blue" ng-click="jobs={};reload()"></i>
+ <i class="ace-icon fa fa-search blue" ng-click="jobList.removeAll();reload()"></i>
</span>
</form>
</div>
@@ -65,12 +65,12 @@
ng-click="toggleSelection(s);" />{{s.name}}
</label>
<!--Refresh Jobs-->
- <button class="btn btn-success btn-xs" style="margin-left: 10px" ng-click="jobs={};reload();"><i class="fa fa-refresh"></i></button>
+ <button class="btn btn-success btn-xs" style="margin-left: 10px" ng-click="jobList.removeAll();reload();"><i class="fa fa-refresh"></i></button>
</div>
</div>
</div>
<!--No Job-->
- <div ng-if="!state.loading && getLength(jobs)==0">
+ <div ng-if="!state.loading && getLength(jobList.jobs)==0">
<div no-result text="No Job."></div>
</div>
<!--Loading Jobs-->
@@ -78,7 +78,7 @@
<loading text="Loading Jobs ..."></loading>
</div>
<!--Jobs Table Content-->
- <table ng-if="getLength(jobs)>0" class="table table-striped table-bordered table-hover dataTable no-footer">
+ <table ng-if="getLength(jobList.jobs)>0" class="table table-striped table-bordered table-hover dataTable no-footer">
<thead>
<tr style="cursor: pointer">
<th ng-repeat="theaditem in jobConfig.theaditems"
@@ -97,7 +97,7 @@
</thead>
<tbody class="odd table table-striped table-bordered table-hover dataTable no-footer">
<tr
- ng-repeat="(uuid,job) in jobs | orderObjectBy:state.filterAttr:state.filterReverse"
+ ng-repeat="(uuid,job) in jobList.jobs | orderObjectBy:state.filterAttr:state.filterReverse"
ng-class="{accordion:true}" ng-click="state.selectedJob = job"
ng-dblclick="state.showSteps= !state.showSteps; state.selectedJob = job"
style="cursor: pointer"
@@ -163,7 +163,7 @@
</table>
</div>
- <kylin-pagination data="jobs" load-func="list" action="action"/>
+ <kylin-pagination data="jobList.jobs" load-func="list" action="action"/>
</div>
<div class="col-xs-4" ng-if="state.showSteps">
<div ng-include src="'partials/jobs/job_steps.html'"></div>
[21/50] incubator-kylin git commit: hll on ii on going
Posted by li...@apache.org.
hll on ii on going
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0a96d742
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0a96d742
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0a96d742
Branch: refs/heads/inverted-index
Commit: 0a96d742771c8a3cafc2f5c54265cbc6531ecf2e
Parents: 9fb8aaa
Author: honma <ho...@ebay.com>
Authored: Wed Feb 11 09:14:20 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Feb 11 09:14:20 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 28 ++++++--
.../org/apache/kylin/common/util/BasicTest.java | 6 +-
.../common/util/HBaseMiniclusterHelper.java | 2 -
.../common/util/HyperLogLogCounterTest.java | 4 +-
.../invertedindex/index/TableRecordInfo.java | 3 +-
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../cardinality/ColumnCardinalityReducer.java | 4 +-
.../job/tools/ColumnCardinalityMapperTest.java | 4 +-
.../job/tools/ColumnCardinalityReducerTest.java | 2 +-
.../kylin/metadata/measure/HLLCSerializer.java | 6 +-
.../measure/fixedlen/FixedHLLCodec.java | 58 ++++++++++++++++
.../measure/fixedlen/FixedLenMeasureCodec.java | 1 +
.../query/sqlfunc/HLLDistinctCountAggFunc.java | 4 +-
.../endpoint/EndpointAggregators.java | 71 +++++++++++++++-----
14 files changed, 149 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index e725e21..c323b90 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -66,7 +66,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
/** The larger p is, the more storage (2^p bytes), the better accuracy */
private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
this.p = p;
- this.m = (int) Math.pow(2, p);
+ this.m = 1 << p;//(int) Math.pow(2, p);
this.hashFunc = hashFunc;
this.registers = new byte[m];
}
@@ -77,7 +77,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
public void add(String value) {
- add(hashFunc.hashString(value,Charset.defaultCharset()).asLong());
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
}
public void add(byte[] value) {
@@ -223,7 +223,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
// the
// moment
- public void writeRegisters(final ByteBuffer out) throws IOException {
+ public void writeCompactRegisters(final ByteBuffer out) throws IOException {
int startPos = out.position();
final int indexLen = getRegisterIndexSize();
@@ -264,7 +264,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
out.put(compressed);
}
- public void readRegisters(ByteBuffer in) throws IOException {
+ public void readCompactRegisters(ByteBuffer in) throws IOException {
byte scheme = in.get();
if ((scheme & COMPRESSION_FLAG) > 0) {
scheme ^= COMPRESSION_FLAG;
@@ -286,12 +286,26 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
registers[key] = in.get();
}
} else { // array scheme
- for (int i = 0; i < m; i++) {
- registers[i] = in.get();
- }
+ in.get(registers);
}
}
+ /**
+ * For compressed output use writeCompactRegisters
+ * @param out
+ */
+ public void writeRegisters(final ByteBuffer out) {
+ out.put(this.registers);
+ }
+
+ /**
+ * For compressed input use readCompactRegisters
+ * @param in
+ */
+ public void readRegisters(ByteBuffer in) {
+ in.get(registers, 0, m);
+ }
+
private int getRegisterIndexSize() {
return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 476b9c1..5952c33 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -64,13 +64,9 @@ public class BasicTest {
public void test1() throws Exception {
}
+
@Test
@Ignore("fix it later")
public void test2() throws IOException, ConfigurationException {
- PropertiesConfiguration a = new PropertiesConfiguration();
- a.setProperty("hi", "dd");
-
- System.out.println("dfads");
- a.save(System.out);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
index 58058ab..3846302 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HBaseMiniclusterHelper.java
@@ -18,7 +18,6 @@
package org.apache.kylin.common.util;
-import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -27,7 +26,6 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.HBaseConnection;
import org.apache.kylin.common.persistence.HBaseResourceStore;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index 75e84c1..088219f 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -108,9 +108,9 @@ public class HyperLogLogCounterTest {
private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
long estimate = hllc.getCountEstimate();
buf.clear();
- hllc.writeRegisters(buf);
+ hllc.writeCompactRegisters(buf);
buf.flip();
- hllc.readRegisters(buf);
+ hllc.readCompactRegisters(buf);
Assert.assertEquals(estimate, hllc.getCountEstimate());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index dca1a65..886c649 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -134,7 +134,8 @@ public class TableRecordInfo {
return -1;
for (int i = 0; i < allColumns.size(); ++i) {
TblColRef tblColRef = allColumns.get(i);
- if (measureSerializers[i] != null && tblColRef.isSameAs(desc.getFactTableName(), metricColumnName)) {
+ if (measureSerializers[i] != null // has measureSerializers means it is a metric
+ && tblColRef.isSameAs(desc.getFactTableName(), metricColumnName)) {
return i;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
index dd7c720..329914a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
@@ -94,7 +94,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWr
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeRegisters(buf);
+ hllc.writeCompactRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit()));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
index a9ac1a3..b4b55d3 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
@@ -50,7 +50,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
for (BytesWritable v : values) {
ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
- hll.readRegisters(buffer);
+ hll.readCompactRegisters(buffer);
getHllc(skey).merge(hll);
hll.clear();
}
@@ -77,7 +77,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeRegisters(buf);
+ hllc.writeCompactRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
// context.write(new Text("ErrorRate_" + key), new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
index e13289a..08c1611 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
@@ -84,7 +84,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readRegisters(ByteBuffer.wrap(bytes));
+ hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
assertTrue(key1 > 0);
assertEquals(8, hllc.getCountEstimate());
}
@@ -117,7 +117,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readRegisters(ByteBuffer.wrap(bytes));
+ hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
System.out.println("ab\177ab".length());
assertTrue(key1 > 0);
assertEquals(1, hllc.getCountEstimate());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
index 034c90e..6ad27a8 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
@@ -67,7 +67,7 @@ public class ColumnCardinalityReducerTest {
}
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeRegisters(buf);
+ hllc.writeCompactRegisters(buf);
buf.flip();
return buf.array();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
index f63c8d7..20c6c21 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
@@ -31,14 +31,14 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
HyperLogLogPlusCounter current;
- HLLCSerializer(int p) {
+ public HLLCSerializer(int p) {
current = new HyperLogLogPlusCounter(p);
}
@Override
public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
try {
- value.writeRegisters(out);
+ value.writeCompactRegisters(out);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -47,7 +47,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
@Override
public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
try {
- current.readRegisters(in);
+ current.readCompactRegisters(in);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
new file mode 100644
index 0000000..9f6a1ba
--- /dev/null
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -0,0 +1,58 @@
+package org.apache.kylin.metadata.measure.fixedlen;
+
+import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.measure.HLLCSerializer;
+import org.apache.kylin.metadata.model.DataType;
+
+import java.util.Map;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 2/10/15.
+ */
+public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> {
+
+ private DataType type;
+ private int presision;
+ private HyperLogLogPlusCounter current;
+
+ public FixedHLLCodec(DataType type) {
+ this.type = type;
+ this.presision = type.getPrecision();
+ this.current = new HyperLogLogPlusCounter(this.presision);
+ }
+
+ @Override
+ public int getLength() {
+ return 1 << presision;
+ }
+
+ @Override
+ public DataType getDataType() {
+ return type;
+ }
+
+ @Override
+ public HyperLogLogPlusCounter valueOf(String value) {
+ current.clear();
+ if (value == null)
+ current.add("__nUlL__");
+ else
+ current.add(value.getBytes());
+ return current;
+ }
+
+ @Override
+ public String toString(HyperLogLogPlusCounter value) {
+ return String.valueOf(value.getCountEstimate());
+ }
+
+ @Override
+ public HyperLogLogPlusCounter read(byte[] buf, int offset) {
+ return serializer.deserialize();
+ }
+
+ @Override
+ public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
index 00de630..41a6356 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedLenMeasureCodec.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.measure.fixedlen;
import org.apache.kylin.metadata.model.DataType;
+
abstract public class FixedLenMeasureCodec<T> {
public static FixedLenMeasureCodec<?> get(DataType type) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
index 356c1e3..8d9ace0 100644
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
+++ b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
@@ -118,12 +118,12 @@ public class HLLDistinctCountAggFunc {
}
@Override
- public void writeRegisters(ByteBuffer out) throws IOException {
+ public void writeCompactRegisters(ByteBuffer out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void readRegisters(ByteBuffer in) throws IOException {
+ public void readCompactRegisters(ByteBuffer in) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0a96d742/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
index 227ecbe..b199862 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase.coprocessor.endpoint;
import com.google.common.collect.Lists;
+import com.yammer.metrics.core.Metric;
import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.invertedindex.index.TableRecordInfo;
@@ -28,6 +29,7 @@ import org.apache.kylin.metadata.measure.MeasureAggregator;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.hbase.coprocessor.CoprocessorConstants;
import org.apache.hadoop.io.LongWritable;
@@ -40,10 +42,28 @@ import java.util.List;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class EndpointAggregators {
+ private enum MetricType {
+ Count, DimensionAsMetric, DistinctCount, Normal
+ }
+
+ private static class MetricInfo {
+ private MetricType type;
+ private int refIndex = -1;
+
+ public MetricInfo(MetricType type, int refIndex) {
+ this.type = type;
+ this.refIndex = refIndex;
+ }
+
+ public MetricInfo(MetricType type) {
+ this.type = type;
+ }
+ }
+
public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) {
String[] funcNames = new String[metrics.size()];
String[] dataTypes = new String[metrics.size()];
- int[] refColIndex = new int[metrics.size()];
+ MetricInfo[] metricInfos = new MetricInfo[metrics.size()];
for (int i = 0; i < metrics.size(); i++) {
FunctionDesc functionDesc = metrics.get(i);
@@ -53,34 +73,40 @@ public class EndpointAggregators {
dataTypes[i] = functionDesc.getReturnType();
if (functionDesc.isCount()) {
- refColIndex[i] = -1;//-1 for count, -2 for metricOnDimension
+ metricInfos[i] = new MetricInfo(MetricType.Count);
} else if (functionDesc.isAppliedOnDimension()) {
- refColIndex[i] = -2;
+ metricInfos[i] = new MetricInfo(MetricType.DimensionAsMetric);
} else {
- refColIndex[i] = tableInfo.findMetric(functionDesc.getParameter().getValue());
- if (refColIndex[i] < 0) {
+ int index = tableInfo.findMetric(functionDesc.getParameter().getValue());
+ if (index < 0) {
throw new IllegalStateException("Column " + functionDesc.getParameter().getColRefs().get(0) + " is not found in II");
}
+
+ if (functionDesc.isCountDistinct()) {
+ metricInfos[i] = new MetricInfo(MetricType.DistinctCount, index);
+ } else {
+ metricInfos[i] = new MetricInfo(MetricType.Normal, index);
+ }
}
}
- return new EndpointAggregators(funcNames, dataTypes, refColIndex, tableInfo.getDigest());
+ return new EndpointAggregators(funcNames, dataTypes, metricInfos, tableInfo.getDigest());
}
final String[] funcNames;
final String[] dataTypes;
- final int[] refColIndex;
+ final MetricInfo[] metricInfos;
final TableRecordInfoDigest tableRecordInfo;
final transient FixedLenMeasureCodec[] measureSerializers;
final transient Object[] metricValues;
- final LongWritable one = new LongWritable(1);
+ final LongWritable ONE = new LongWritable(1);
- public EndpointAggregators(String[] funcNames, String[] dataTypes, int[] refColIndex, TableRecordInfoDigest tableInfo) {
+ public EndpointAggregators(String[] funcNames, String[] dataTypes, MetricInfo[] metricInfos, TableRecordInfoDigest tableInfo) {
this.funcNames = funcNames;
this.dataTypes = dataTypes;
- this.refColIndex = refColIndex;
+ this.metricInfos = metricInfos;
this.tableRecordInfo = tableInfo;
this.metricValues = new Object[funcNames.length];
@@ -111,12 +137,19 @@ public class EndpointAggregators {
int rawIndex = 0;
int columnCount = tableRecordInfo.getColumnCount();
- //normal column values to aggregate
for (int columnIndex = 0; columnIndex < columnCount; ++columnIndex) {
- if (tableRecordInfo.isMetrics(columnIndex)) {
- for (int metricIndex = 0; metricIndex < refColIndex.length; ++metricIndex) {
- if (refColIndex[metricIndex] == columnIndex) {
+ for (int metricIndex = 0; metricIndex < metricInfos.length; ++metricIndex) {
+ if (metricInfos[metricIndex].refIndex == columnIndex) {
+ if (metricInfos[metricIndex].type == MetricType.Normal) {
+ //normal column values to aggregate
measureAggrs[metricIndex].aggregate(measureSerializers[metricIndex].read(row, rawIndex));
+ } else if (metricInfos[metricIndex].type == MetricType.DistinctCount) {
+ if (tableRecordInfo.isMetrics(columnCount)) {
+ measureAggrs[metricIndex].aggregate(measureSerializers[metricIndex].read(row, rawIndex));
+ } else {
+ //TODO: for unified dictionary, this is okay. but if different data blocks uses different dictionary, we'll have to aggregate original data
+ measureAggrs[metricIndex].aggregate(tableRecordInfo.);
+ }
}
}
}
@@ -124,9 +157,11 @@ public class EndpointAggregators {
}
//aggregate for "count"
- for (int i = 0; i < refColIndex.length; ++i) {
- if (refColIndex[i] == -1) {
- measureAggrs[i].aggregate(one);
+ for (int i = 0; i < metricInfos.length; ++i) {
+ if (metricInfos[i].type == MetricType.Count) {
+ measureAggrs[i].aggregate(ONE);
+ } else if (metricInfos[i].type == MetricType.DistinctCount) {
+
}
}
}
@@ -180,7 +215,7 @@ public class EndpointAggregators {
public void serialize(EndpointAggregators value, ByteBuffer out) {
BytesUtil.writeAsciiStringArray(value.funcNames, out);
BytesUtil.writeAsciiStringArray(value.dataTypes, out);
- BytesUtil.writeIntArray(value.refColIndex, out);
+ BytesUtil.writeIntArray(value.metricInfos, out);
BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfo), out);
}
[23/50] incubator-kylin git commit: fix script
Posted by li...@apache.org.
fix script
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/78689e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/78689e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/78689e34
Branch: refs/heads/inverted-index
Commit: 78689e34d77e12145433a45fbab4ba3d1832fd6b
Parents: 09f4bc3
Author: qianhao.zhou <qi...@ebay.com>
Authored: Wed Feb 11 11:49:12 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Wed Feb 11 11:49:12 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 3 ---
script/compress.sh | 7 ++++---
script/package.sh | 2 +-
script/prepare.sh | 4 ++--
4 files changed, 7 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78689e34/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
index 735de53..9225fb6 100755
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
@@ -31,9 +31,6 @@ spring_profile="default"
if [ "$useSandbox" = "true" ]
then spring_profile="sandbox"
fi
-rm -rf ${tomcat_root}/webapps/kylin*
-cp ${dir}/../lib/kylin-server-*.war ${tomcat_root}/webapps/kylin.war
-
source ${dir}/find-hive-dependency.sh
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78689e34/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index 6221b2b..01a8376 100644
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -13,6 +13,7 @@ fi
#package tgz
echo 'package tgz'
-tar -cvzf kylin-${version}.tgz lib bin conf tomcat
-
-rm -rf tomcat/webapps/*
+mkdir kylin-${version}
+cp -r lib bin conf tomcat kylin-${version}
+tar -cvzf kylin-${version}.tgz kylin-${version}
+rm -rf kylin-${version}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78689e34/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
index 018491e..556467a 100755
--- a/script/package.sh
+++ b/script/package.sh
@@ -26,8 +26,8 @@ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpress
echo "kylin version: ${version}"
export version
+sh ${dir}/download-tomcat.sh || { exit 1; }
sh ${dir}/build.sh || { exit 1; }
sh ${dir}/prepare.sh || { exit 1; }
-sh ${dir}/download-tomcat.sh || { exit 1; }
sh ${dir}/compress.sh || { exit 1; }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/78689e34/script/prepare.sh
----------------------------------------------------------------------
diff --git a/script/prepare.sh b/script/prepare.sh
index 8a855e1..3b30ac2 100644
--- a/script/prepare.sh
+++ b/script/prepare.sh
@@ -13,7 +13,7 @@ fi
echo "copy lib file"
rm -rf lib
mkdir lib
-cp server/target/kylin-server-${version}.war lib
+cp server/target/kylin-server-${version}.war tomcat/webapps/kylin.war
cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
@@ -28,5 +28,5 @@ cd webapp/dist
for f in * .[^.]*
do
echo "Adding $f to war"
- jar -uf ../../lib/kylin-server-${version}.war $f
+ jar -uf ../../tomcat/webapps/kylin.war $f
done
\ No newline at end of file
[09/50] incubator-kylin git commit: fix
mapreduce.application.classpath
Posted by li...@apache.org.
fix mapreduce.application.classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/40af1c82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/40af1c82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/40af1c82
Branch: refs/heads/inverted-index
Commit: 40af1c82ff4dc7989de26a3dc949196e8c287801
Parents: 6e53c4d
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 16:53:23 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 16:53:23 2015 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox-hdp21/mapred-site.xml | 2 +-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40af1c82/examples/test_case_data/sandbox-hdp21/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/mapred-site.xml b/examples/test_case_data/sandbox-hdp21/mapred-site.xml
index a0c6409..b699c5d 100644
--- a/examples/test_case_data/sandbox-hdp21/mapred-site.xml
+++ b/examples/test_case_data/sandbox-hdp21/mapred-site.xml
@@ -98,7 +98,7 @@
</property>
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,/usr/lib/hive/conf/,/usr/lib/hbase/lib/*,/usr/lib/hive-hcatalog/share/hcatalog/*,/usr/lib/hive/lib/*</value>
+ <value>/usr/lib/hive/conf/,/usr/lib/hbase/lib/*,/usr/lib/hive-hcatalog/share/hcatalog/*,/usr/lib/hive/lib/*</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/40af1c82/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index ee3837c..ccede13 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -43,7 +43,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*,$PWD/mr-framework/hadoop/share/hadoop/common/*,$PWD/mr-framework/hadoop/share/hadoop/common/lib/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar,/etc/hadoop/conf/secure</value>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*,$PWD/mr-framework/hadoop/share/hadoop/common/*,$PWD/mr-framework/hadoop/share/hadoop/common/lib/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/*,$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/*,$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar,/etc/hadoop/conf/secure</value>
</property>
<property>
[14/50] incubator-kylin git commit: fix bin
Posted by li...@apache.org.
fix bin
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/681314c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/681314c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/681314c3
Branch: refs/heads/inverted-index
Commit: 681314c3ab664d0944cdf85c56b12591dd11976c
Parents: cb1de27
Author: qianhao.zhou <qi...@ebay.com>
Authored: Tue Feb 10 17:35:21 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Tue Feb 10 17:35:21 2015 +0800
----------------------------------------------------------------------
bin/start-kylin.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/681314c3/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
index d328851..a6aabdf 100755
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
@@ -28,7 +28,7 @@ sh ${dir}/check-env.sh || { exit 1; }
useSandbox=`cat ${KYLIN_HOME}/conf/kylin.properties | grep 'kylin.sandbox' | awk -F '=' '{print $2}'`
spring_profile="default"
-if [ "$useSandbox" -eq true ]
+if [ "$useSandbox" = "true" ]
then spring_profile="sandbox"
fi
rm -rf ${tomcat_root}/webapps/kylin*
@@ -51,7 +51,7 @@ hbase -Djava.util.logging.config.file=${tomcat_root}/conf/logging.properties \
-Dspring.profiles.active=${spring_profile} \
org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin_sandbox.log 2>&1 &
echo "A new Kylin instance is started by $USER, stop it using \"stop-kylin.sh\""
-if [ "$useSandbox" -eq true ]
+if [ "$useSandbox" = "true" ]
then echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
else
echo "Please visit http://<ip>:7070/kylin"
[29/50] incubator-kylin git commit: minor formatting
Posted by li...@apache.org.
minor formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/6ef8be00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/6ef8be00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/6ef8be00
Branch: refs/heads/inverted-index
Commit: 6ef8be00509f554cee4b2707fc1a8180b6aba129
Parents: 75431e3
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:45:01 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:45:01 2015 +0800
----------------------------------------------------------------------
.../kylin/common/hll/HyperLogLogPlusCounter.java | 16 +++-------------
1 file changed, 3 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/6ef8be00/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index e725e21..18b3af1 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -77,7 +77,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
public void add(String value) {
- add(hashFunc.hashString(value,Charset.defaultCharset()).asLong());
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
}
public void add(byte[] value) {
@@ -209,19 +209,9 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
};
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
- // disable
- // due to
- // slowness
+ public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes, disable due to slowness
public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
- // lib
- // has
- // a
- // bug
- // at
- // the
- // moment
+ public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF lib has a bug at the moment
public void writeRegisters(final ByteBuffer out) throws IOException {
int startPos = out.position();
[40/50] incubator-kylin git commit: Merge pull request #424 from
janzhongi/inverted-index
Posted by li...@apache.org.
Merge pull request #424 from janzhongi/inverted-index
add JobList &CubeList service
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d5a185e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d5a185e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d5a185e3
Branch: refs/heads/inverted-index
Commit: d5a185e3b2ec195ac53c8810c9d7a0e95cc495a8
Parents: 8245337 2afafb2
Author: Zhong,Jian <ji...@ebay.com>
Authored: Wed Feb 11 19:55:07 2015 +0800
Committer: Zhong,Jian <ji...@ebay.com>
Committed: Wed Feb 11 19:55:07 2015 +0800
----------------------------------------------------------------------
.gitignore | 0
bin/check-env.sh | 0
bin/start-kylin.sh | 0
bin/stop-kylin.sh | 0
conf/kylin.properties | 0
.../sandbox-hdp21/kylin.properties | 0
.../test_case_data/sandbox/kylin.properties | 0
script/package.sh | 0
webapp/app/css/AdminLTE-fonts.css | 0
webapp/app/css/AdminLTE.css | 0
...1aHBYDBqgeIAH2zlPytCVloACqnDHJwh-em2k8.woff2 | Bin
...t5NIMarvNW-hkYWvT3rGVtsTkPsbDajuO5ueQw.woff2 | Bin
webapp/app/index.html | 2 +
webapp/app/js/controllers/cubeAdvanceSetting.js | 0
webapp/app/js/controllers/cubeDimensions.js | 0
webapp/app/js/controllers/cubeEdit.js | 0
webapp/app/js/controllers/cubeFilter.js | 0
webapp/app/js/controllers/cubeModel.js | 0
webapp/app/js/controllers/cubeRefresh.js | 0
webapp/app/js/controllers/cubeSchema.js | 0
webapp/app/js/controllers/cubes.js | 51 +++++----------
webapp/app/js/controllers/job.js | 41 ++++--------
webapp/app/js/controllers/page.js | 0
webapp/app/js/controllers/projects.js | 0
webapp/app/js/controllers/sourceMeta.js | 0
webapp/app/js/filters/filter.js | 0
webapp/app/js/model/cubeConfig.js | 0
webapp/app/js/model/cubeListModel.js | 65 +++++++++++++++++++
webapp/app/js/model/cubesManager.js | 0
webapp/app/js/model/jobConfig.js | 0
webapp/app/js/model/jobListModel.js | 61 +++++++++++++++++
webapp/app/js/model/metaModel.js | 0
webapp/app/js/model/projectConfig.js | 0
webapp/app/js/model/projectModel.js | 0
webapp/app/js/model/tableConfig.js | 0
webapp/app/js/model/tableModel.js | 0
webapp/app/partials/common/access.html | 8 ++-
.../cubeDesigner/advanced_settings.html | 3 +
.../app/partials/cubeDesigner/data_model.html | 1 +
.../app/partials/cubeDesigner/dimensions.html | 3 +
.../app/partials/cubeDesigner/incremental.html | 1 +
webapp/app/partials/cubeDesigner/measures.html | 2 +
webapp/app/partials/cubes/cube_json_edit.html | 0
webapp/app/partials/cubes/cubes.html | 14 ++--
webapp/app/partials/jobs/job_submit.html | 1 -
webapp/app/partials/jobs/jobs.html | 14 ++--
webapp/app/partials/projects/projects.html | 0
webapp/app/partials/tables/source_metadata.html | 0
webapp/bower.json | 0
webapp/grunt.json | 0
50 files changed, 187 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
[39/50] incubator-kylin git commit: fix select bug KYLIN-519
Posted by li...@apache.org.
fix select bug KYLIN-519
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2afafb21
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2afafb21
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2afafb21
Branch: refs/heads/inverted-index
Commit: 2afafb21598f2535756f1dcf56b74ac73f5b7b9c
Parents: 285bd56
Author: jiazhong <ji...@ebay.com>
Authored: Wed Feb 11 19:53:55 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Wed Feb 11 19:53:55 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/common/access.html | 8 ++++++--
webapp/app/partials/cubeDesigner/advanced_settings.html | 3 +++
webapp/app/partials/cubeDesigner/data_model.html | 1 +
webapp/app/partials/cubeDesigner/dimensions.html | 3 +++
webapp/app/partials/cubeDesigner/incremental.html | 1 +
webapp/app/partials/cubeDesigner/measures.html | 2 ++
6 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/common/access.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/common/access.html b/webapp/app/partials/common/access.html
index d3ef769..74f8c9f 100644
--- a/webapp/app/partials/common/access.html
+++ b/webapp/app/partials/common/access.html
@@ -43,12 +43,16 @@
<input ng-model="newAccess.sid" ng-if="newAccess.principal==true" placeholder=" User NT Account..." style="width: 80%" />
<select chosen ng-model="newAccess.sid" ng-if="newAccess.principal==false" style="width: 80%"
- ng-options="authority as authority for authority in authorities"></select>
+ ng-options="authority as authority for authority in authorities">
+ <option value=""></option>
+ </select>
</td>
<td >
<label><b>Permission </b> </label>
<select ng-model="newAccess.permission" ng-init="newAccess.permission=permissions.READ.value"
- ng-options="permission.value as permission.name for (name, permission) in permissions"></select>
+ ng-options="permission.value as permission.name for (name, permission) in permissions">
+ <option value=""></option>
+ </select>
</td>
<td>
<button class="btn btn-default btn-xs" ng-click="grant(type, entity, newAccess)"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 53aac03..5c65886 100644
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -33,6 +33,7 @@
<select class="form-control" ng-if="state.mode=='edit'" style="width: 360px;"
chosen ng-model="metaModel.model.capacity" required
ng-options="cc as cc for cc in cubeConfig.cubeCapacities">
+ <option value=""></option>
</select>
<span ng-if="state.mode=='view'">{{metaModel.model.capacity}}</span>
</div>
@@ -65,6 +66,7 @@
ng-model="aggregation_groups" required chosen multiple
ng-change="refreshAggregationGroup(cubeMetaFrame.rowkey.aggregation_groups, $index, aggregation_groups)"
ng-options="rowkey_column.column as rowkey_column.column for rowkey_column in cubeMetaFrame.rowkey.rowkey_columns">
+ <option value=""></option>
</select>
<p ng-if="state.mode=='view'" style="word-wrap: break-word; word-break: normal;max-width: 1000px ">{{aggregation_groups}}</p>
</td>
@@ -126,6 +128,7 @@
<select ng-if="state.mode=='edit'" style="width:80px;"
chosen ng-model="rowkey_column.dictionary"
ng-options="dt as dt for dt in cubeConfig.dictionaries">
+ <option value=""></option>
</select>
<span ng-if="state.mode=='view'">{{rowkey_column.dictionary}}</span>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/cubeDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/data_model.html b/webapp/app/partials/cubeDesigner/data_model.html
index aa221b6..bdfda69 100644
--- a/webapp/app/partials/cubeDesigner/data_model.html
+++ b/webapp/app/partials/cubeDesigner/data_model.html
@@ -125,6 +125,7 @@
<div class="col-sm-6">
<select class="form-control" chosen ng-model="newLookup.join.type"
ng-options="joinType.value as joinType.name for joinType in cubeConfig.joinTypes">
+ <option value=""></option>
</select>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index f182e11..21ef2f5 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -161,11 +161,13 @@
<div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') >= 0">
<select class="form-control" required="true" chosen data-placeholder="Select a lookup table"
ng-model="newDimension.table" ng-options="table for table in availableTables.slice(1)">
+ <option value=""></option>
</select>
</div>
<div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') == -1">
<select class="form-control" required="true" chosen data-placeholder="Select a table"
ng-model="newDimension.table" ng-options="table for table in availableTables">
+ <option value=""></option>
</select>
</div>
<div class="col-xs-12 col-sm-3">
@@ -183,6 +185,7 @@
<div class="col-xs-12 col-sm-6">
<select class="form-control" required="true" chosen data-placeholder="Select a column"
ng-model="newDimension.column[0]" ng-options="column.name as column.name for column in getColumnsByTable(newDimension.table)">
+ <option value=""></option>
</select>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/cubeDesigner/incremental.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/incremental.html b/webapp/app/partials/cubeDesigner/incremental.html
index c8dd927..b3a6b1e 100644
--- a/webapp/app/partials/cubeDesigner/incremental.html
+++ b/webapp/app/partials/cubeDesigner/incremental.html
@@ -29,6 +29,7 @@
ng-if="state.mode=='edit'"
chosen ng-model="metaModel.model.partition_desc.partition_type"
ng-options="ddt as ddt for ddt in cubeConfig.cubePartitionTypes">
+ <option value=""></option>
</select>
<span ng-if="state.mode=='view'">{{metaModel.model.partition_desc.partition_type}}</span>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2afafb21/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 7599d51..7e19167 100644
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -112,6 +112,7 @@
<select class="form-control"
ng-init="newMeasure.function.expression = (!!newMeasure.function.expression)?newMeasure.function.expression:cubeConfig.dftSelections.measureExpression" chosen ng-model="newMeasure.function.expression" required
ng-options="me as me for me in cubeConfig.measureExpressions">
+ <option value=""></option>
</select>
</div>
</div>
@@ -125,6 +126,7 @@
ng-init="newMeasure.function.parameter.type=(!!newMeasure.function.parameter.type)?newMeasure.function.parameter.type: 'column' "
chosen ng-model="newMeasure.function.parameter.type" required
ng-options="mpt as mpt for mpt in cubeConfig.measureParamType">
+ <option value=""></option>
</select>
<span class="font-color-default"
ng-if="newMeasure.function.expression == 'COUNT'"