You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/12 06:43:31 UTC
[41/50] incubator-kylin git commit: add installation guide
add installation guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/56ba5131
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/56ba5131
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/56ba5131
Branch: refs/heads/inverted-index
Commit: 56ba5131a780ce3ada62c9068efab49b67e679b1
Parents: 443b192
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Feb 12 10:35:46 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Thu Feb 12 10:35:46 2015 +0800
----------------------------------------------------------------------
bin/health-check.sh | 33 ++++++++++
bin/healthmon.sh | 33 ----------
.../org/apache/kylin/common/KylinConfig.java | 3 +-
conf/kylin.properties | 8 +--
docs/Installation/install_and_run.md | 69 ++++++++++++++++++++
.../test_case_data/localmeta/kylin.properties | 6 --
.../localmeta_v1/kylin.properties | 6 --
.../minicluster/b-kylin/kylin.properties | 6 --
.../minicluster/b-kylin/meta/kylin.properties | 6 --
.../test_case_data/minicluster/kylin.properties | 6 --
.../sandbox-hdp21/kylin.properties | 6 --
.../test_case_data/sandbox/kylin.properties | 6 --
server/ServiceConfig.md | 4 +-
13 files changed, 105 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/health-check.sh
----------------------------------------------------------------------
diff --git a/bin/health-check.sh b/bin/health-check.sh
new file mode 100644
index 0000000..439eef4
--- /dev/null
+++ b/bin/health-check.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+ALERT="your@email.com"
+
+OUTPUT=$(
+ curl --max-time 20 -# \
+ --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
+ -H "Authorization:Basic QURNSU46S1lMSU4=" \
+ -H "Content-Type:application/json;charset=UTF-8" \
+ http://localhost:7070/kylin/api/query \
+)
+
+# ----------------------------------------------------------------------------
+
+date
+
+if [[ $OUTPUT == *"results"* ]]; then
+ echo "Good."
+else
+ echo "Bad."
+ TS_FILE=/tmp/kylin_healthmon_ts
+ LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
+ CURR_TS=`date +%s`
+ echo last: $LAST_TS
+ echo curr: $CURR_TS
+ if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
+ echo "Sending mail..."
+ echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
+ if [ "$?" == "0" ]; then
+ touch $TS_FILE
+ fi
+ fi
+fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/bin/healthmon.sh
----------------------------------------------------------------------
diff --git a/bin/healthmon.sh b/bin/healthmon.sh
deleted file mode 100644
index 439eef4..0000000
--- a/bin/healthmon.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-ALERT="your@email.com"
-
-OUTPUT=$(
- curl --max-time 20 -# \
- --data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
- -H "Authorization:Basic QURNSU46S1lMSU4=" \
- -H "Content-Type:application/json;charset=UTF-8" \
- http://localhost:7070/kylin/api/query \
-)
-
-# ----------------------------------------------------------------------------
-
-date
-
-if [[ $OUTPUT == *"results"* ]]; then
- echo "Good."
-else
- echo "Bad."
- TS_FILE=/tmp/kylin_healthmon_ts
- LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
- CURR_TS=`date +%s`
- echo last: $LAST_TS
- echo curr: $CURR_TS
- if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
- echo "Sending mail..."
- echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
- if [ "$?" == "0" ]; then
- touch $TS_FILE
- fi
- fi
-fi
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index ef2c6e2..add87e0 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -89,9 +89,8 @@ public class KylinConfig {
public static final String KYLIN_JOB_JAR = "kylin.job.jar";
public static final String COPROCESSOR_LOCAL_JAR = "kylin.coprocessor.local.jar";
- public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
- public static final String KYLIN_JOB_JAR_LOCAL = "kylin.job.jar.local";
+ public static final String COPROCESSOR_SCAN_BITS_THRESHOLD = "kylin.coprocessor.scan.bits.threshold";
public static final String KYLIN_JOB_LOG_DIR = "kylin.job.log.dir";
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index bf86a26..f607dca 100755
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -4,20 +4,14 @@
kylin.rest.servers=
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata_qa@hbase
+kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/docs/Installation/install_and_run.md
----------------------------------------------------------------------
diff --git a/docs/Installation/install_and_run.md b/docs/Installation/install_and_run.md
new file mode 100644
index 0000000..d634b32
--- /dev/null
+++ b/docs/Installation/install_and_run.md
@@ -0,0 +1,69 @@
+##Install and Run
+
+###How to run
+
+1. Download the release version(according to the hadoop distribution)
+2. Setup a KYLIN_HOME pointing to the corresponding directory where you extract the release tar
+3. Make sure the user has the privilege to run hadoop, hive and hbase cmd in shell. If you are not so sure, you can just run **bin/check-env.sh**, it will print out the detail information if you have some environment issues.
+4. To start Kylin, simply run **bin/start-kylin.sh**
+5. To stop Kylin, simply run **bin/stop-kylin.sh**
+
+
+If you are running Kylin in a cluster or you have multiple Kylin instances, please make sure you have the following property correctly configured.
+
+1. kylin.rest.servers
+
+ List of web servers in use, this enables one web server instance to sync up with other servers.
+
+
+2. kylin.server.mode
+
+ Make sure there is only one instance whose "kylin.server.mode" is set to "all" if there are multiple instances.
+
+
+###The directory structure of the installation
+
+> ├── bin
+> │ ├── check-env.sh
+> │ ├── find-hive-dependency.sh
+> │ ├── health-check.sh
+> │ ├── start-kylin.sh
+> │ └── stop-kylin.sh
+> │
+> ├── conf
+> │ ├── kylin_job_conf.xml
+> │ └── kylin.properties
+> │
+> ├── tomcat
+> │ ├── webapps
+> │ │ └── kylin.war
+> │ …….
+> │
+> └── lib
+> ├── kylin-coprocessor-${version}.jar
+> └── kylin-job-${version}.jar
+
+
+
+###Build from source
+> git clone https://github.com/KylinOLAP/Kylin.git
+> cd KylinOLAP/Kylin
+> sh script/package.sh
+
+In order to generate binary package, **maven** and **npm** are pre-requisites.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 53d24ee..aa18287 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-coprocessor-latest.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/localmeta_v1/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta_v1/kylin.properties b/examples/test_case_data/localmeta_v1/kylin.properties
index 96439b4..aa18287 100644
--- a/examples/test_case_data/localmeta_v1/kylin.properties
+++ b/examples/test_case_data/localmeta_v1/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties
index 19c8de6..5d1b667 100644
--- a/examples/test_case_data/minicluster/b-kylin/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
index 419b9e8..17e5c99 100644
--- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
+++ b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index 7816cad..373186c 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:minicluster:/hbase-unsecure
# The storage for final cube file in hbase
kylin.storage.url=hbase:minicluster:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox-hdp21/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox-hdp21/kylin.properties b/examples/test_case_data/sandbox-hdp21/kylin.properties
index 7474283..3e2f97a 100755
--- a/examples/test_case_data/sandbox-hdp21/kylin.properties
+++ b/examples/test_case_data/sandbox-hdp21/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-u
# The storage for final cube file in hbase
kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=/tmp/kylin/kylin-job-latest-job.jar
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index adc0dda..cd5d02c 100755
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -9,15 +9,9 @@ kylin.metadata.url=kylin_metadata_qa@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
-kylin.coprocessor.local.jar=
-
# Temp folder in hdfs
kylin.hdfs.working.dir=/tmp
-# Path to the local(relative to job engine) job jar, job engine will use this jar
-kylin.job.jar=
-
kylin.job.mapreduce.default.reduce.input.mb=500
# If true, job engine will not assume that hadoop CLI reside on the same server as it self
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/56ba5131/server/ServiceConfig.md
----------------------------------------------------------------------
diff --git a/server/ServiceConfig.md b/server/ServiceConfig.md
index 0f951be..fdeba33 100644
--- a/server/ServiceConfig.md
+++ b/server/ServiceConfig.md
@@ -20,8 +20,6 @@
##### kylin.query.cache.enabled
### Job
-##### kylin.job.jar
-##### kylin.job.log.dir
##### kylin.job.hdfs.working.dir
##### kylin.job.mapreduce.default.reduce.input.mb
##### kylin.job.mapreduce.default.reduce.count.ratio
@@ -35,6 +33,6 @@
##### kylin.job.concurrent.max.limit
##### kylin.job.command.exe.type
##### kylin.job.admin.dls
-##### kylin.job.step.timeout
+##### kylin.job.step.timeoutww
##### kylin.job.yarn.app.rest.check.interval.seconds
##### kylin.job.concurrent.max.limit
\ No newline at end of file