You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/07/22 04:08:28 UTC
[1/5] kylin git commit: Find a better way to check hadoop job status
via job API
Repository: kylin
Updated Branches:
refs/heads/tpy [created] 75f924705
Find a better way to check hadoop job status via job API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8283b76a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8283b76a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8283b76a
Branch: refs/heads/tpy
Commit: 8283b76a8c7acc8afd1cd31ff9dd8959a240f6c2
Parents: cf4d294
Author: kyotoYaho <nj...@apache.org>
Authored: Wed May 4 14:48:44 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jul 7 11:55:35 2016 +0800
----------------------------------------------------------------------
.../mr/common/HadoopJobStatusChecker.java | 64 ++++++++++++++++++++
.../engine/mr/common/MapReduceExecutable.java | 18 +++---
2 files changed, 73 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8283b76a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java
new file mode 100644
index 0000000..f25d41f
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopJobStatusChecker.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.kylin.job.constant.JobStepStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HadoopJobStatusChecker {
+
+ protected static final Logger logger = LoggerFactory.getLogger(HadoopJobStatusChecker.class);
+
+ public static JobStepStatusEnum checkStatus(Job job, StringBuilder output) {
+ if (job == null || job.getJobID() == null) {
+ output.append("Skip status check with empty job id..\n");
+ return JobStepStatusEnum.WAITING;
+ }
+
+ JobStepStatusEnum status = null;
+ try {
+ switch (job.getStatus().getState()) {
+ case SUCCEEDED:
+ status = JobStepStatusEnum.FINISHED;
+ break;
+ case FAILED:
+ status = JobStepStatusEnum.ERROR;
+ break;
+ case KILLED:
+ status = JobStepStatusEnum.KILLED;
+ break;
+ case RUNNING:
+ status = JobStepStatusEnum.RUNNING;
+ break;
+ case PREP:
+ status = JobStepStatusEnum.WAITING;
+ break;
+ }
+ } catch (Exception e) {
+ logger.error("error check status", e);
+ output.append("Exception: " + e.getLocalizedMessage() + "\n");
+ status = JobStepStatusEnum.ERROR;
+ }
+
+ return status;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8283b76a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index aefcb95..4cf7dfb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -137,17 +137,17 @@ public class MapReduceExecutable extends AbstractExecutable {
final StringBuilder output = new StringBuilder();
final HadoopCmdOutput hadoopCmdOutput = new HadoopCmdOutput(job, output);
- final String restStatusCheckUrl = getRestStatusCheckUrl(job, context.getConfig());
- if (restStatusCheckUrl == null) {
- logger.error("restStatusCheckUrl is null");
- return new ExecuteResult(ExecuteResult.State.ERROR, "restStatusCheckUrl is null");
- }
- String mrJobId = hadoopCmdOutput.getMrJobId();
- boolean useKerberosAuth = context.getConfig().isGetJobStatusWithKerberos();
- HadoopStatusChecker statusChecker = new HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
+// final String restStatusCheckUrl = getRestStatusCheckUrl(job, context.getConfig());
+// if (restStatusCheckUrl == null) {
+// logger.error("restStatusCheckUrl is null");
+// return new ExecuteResult(ExecuteResult.State.ERROR, "restStatusCheckUrl is null");
+// }
+// String mrJobId = hadoopCmdOutput.getMrJobId();
+// boolean useKerberosAuth = context.getConfig().isGetJobStatusWithKerberos();
+// HadoopStatusChecker statusChecker = new HadoopStatusChecker(restStatusCheckUrl, mrJobId, output, useKerberosAuth);
JobStepStatusEnum status = JobStepStatusEnum.NEW;
while (!isDiscarded()) {
- JobStepStatusEnum newStatus = statusChecker.checkStatus();
+ JobStepStatusEnum newStatus = HadoopJobStatusChecker.checkStatus(job, output);
if (status == JobStepStatusEnum.KILLED) {
executableManager.updateJobOutput(getId(), ExecutableState.ERROR, Collections.<String, String> emptyMap(), "killed by admin");
return new ExecuteResult(ExecuteResult.State.FAILED, "killed by admin");
[4/5] kylin git commit: KYLIN-1795: fix sample.sh when using beeline
Posted by sh...@apache.org.
KYLIN-1795: fix sample.sh when using beeline
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df3ec7e6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df3ec7e6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df3ec7e6
Branch: refs/heads/tpy
Commit: df3ec7e64f28fe93ba43eec51494260f056894db
Parents: 39b3b35
Author: Yiming Liu <li...@gmail.com>
Authored: Fri Jul 22 09:36:24 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jul 22 10:27:36 2016 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 8 +++++++-
examples/sample_cube/create_sample_tables.sql | 6 +++---
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/df3ec7e6/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 7cc50c5..9939c62 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -25,6 +25,10 @@ job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
echo "Going to create sample tables in hive..."
cd ${KYLIN_HOME}/sample_cube/data
+echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
+hadoop fs -mkdir -p /tmp/kylin/sample_cube/data
+hadoop fs -put * /tmp/kylin/sample_cube/data/
+
hive_client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.client`
if [ "${hive_client_mode}" == "beeline" ]
then
@@ -35,6 +39,8 @@ else
fi
echo "Sample hive tables are created successfully; Going to create sample cube..."
+hadoop fs -rm -r /tmp/kylin/sample_cube
+
cd ${KYLIN_HOME}
hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
-echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
\ No newline at end of file
+echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
http://git-wip-us.apache.org/repos/asf/kylin/blob/df3ec7e6/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 943c0fa..d83566c 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -185,6 +185,6 @@ TRANS_ID bigint
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
-LOAD DATA LOCAL INPATH 'DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
\ No newline at end of file
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
\ No newline at end of file
[2/5] kylin git commit: KYLIN-1679 get hive bee line properties fixed
Posted by sh...@apache.org.
KYLIN-1679 get hive bee line properties fixed
Signed-off-by: zkld123 <zk...@sjtu.edu.cn>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b8cae1ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b8cae1ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b8cae1ec
Branch: refs/heads/tpy
Commit: b8cae1ec24aefe586fc1a4656745d6264c129f81
Parents: 8283b76
Author: Lingyan Jiang <ly...@hotmail.com>
Authored: Thu Jun 16 15:27:57 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jul 22 10:23:24 2016 +0800
----------------------------------------------------------------------
build/bin/get-properties.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b8cae1ec/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
old mode 100644
new mode 100755
index 4c93b3b..3151ad6
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -23,8 +23,9 @@ then
exit -1
fi
+IFS=$'\n'
result=
-for i in `cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "$1" | grep -v '^#' |awk -F '=' '{print $2}' | cut -c 1-`
+for i in `cat ${KYLIN_HOME}/conf/kylin.properties|grep -w "$1"| grep -v '^#' |awk -F= '{ n = index($0,"="); print substr($0,n+1)}' | cut -c 1-`
do
:
result=$i
[5/5] kylin git commit: KYLIN-1912 print sql with beeline
Posted by sh...@apache.org.
KYLIN-1912 print sql with beeline
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75f92470
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75f92470
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75f92470
Branch: refs/heads/tpy
Commit: 75f924705a1593e44b7a07c4e5d09753aa1e1054
Parents: df3ec7e
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jul 22 11:45:16 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jul 22 11:45:16 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/source/hive/HiveCmdBuilder.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/75f92470/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
index 5a8278e..bab902d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveCmdBuilder.java
@@ -18,10 +18,7 @@
package org.apache.kylin.source.hive;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.*;
import java.util.ArrayList;
import org.apache.commons.io.IOUtils;
@@ -78,6 +75,9 @@ public class HiveCmdBuilder {
buf.append(tmpHql.getAbsolutePath());
logger.info("The statements to execute in beeline: \n" + hqlBuf);
+ if (logger.isDebugEnabled()) {
+ logger.debug("The SQL to execute in beeline: \n" + IOUtils.toString(new FileReader(tmpHql)));
+ }
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
[3/5] kylin git commit: KYLIN-1912 support for beeline
Posted by sh...@apache.org.
KYLIN-1912 support for beeline
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/39b3b351
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/39b3b351
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/39b3b351
Branch: refs/heads/tpy
Commit: 39b3b35141cdc4feb5cce323a5a4c70496b5ab4a
Parents: b8cae1e
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jul 22 10:27:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jul 22 10:27:19 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/39b3b351/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 4026006..24c5705 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -177,7 +177,7 @@ public class HiveMRInput implements IMRInput {
} catch (IOException e) {
throw new RuntimeException("Failed to generate hive set statements for createFlatHiveTableStep", e);
}
- final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
+ final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
hiveCmdBuilder.addStatement(useDatabaseHql);
hiveCmdBuilder.addStatement(setHql);
for(TableDesc lookUpTableDesc : lookupViewsTables) {