You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/16 09:30:38 UTC
[1/2] incubator-kylin git commit: KYLIN-1149 When yarn return an
incomplete job tracking URL, Kylin will fail to get job status
Repository: incubator-kylin
Updated Branches:
refs/heads/1.x-staging 2432e2d6f -> 1afb32dc5
KYLIN-1149 When yarn return an incomplete job tracking URL, Kylin will fail to get job status
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/db7f8be0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/db7f8be0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/db7f8be0
Branch: refs/heads/1.x-staging
Commit: db7f8be0fe2929edd72fff0ea4b0fcfe937816e5
Parents: 2432e2d
Author: shaofengshi <sh...@apache.org>
Authored: Mon Nov 16 16:29:16 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Nov 16 16:29:16 2015 +0800
----------------------------------------------------------------------
.../kylin/job/tools/HadoopStatusGetter.java | 21 ++++++++++++++++++++
1 file changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/db7f8be0/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 398d15a..0cd4d43 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -19,6 +19,7 @@
package org.apache.kylin.job.tools;
import java.io.IOException;
+import java.net.MalformedURLException;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
@@ -26,6 +27,7 @@ import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -84,6 +86,11 @@ public class HadoopStatusGetter {
int cut = s.indexOf("url=");
if (cut >= 0) {
redirect = s.substring(cut + 4);
+
+ if (isValidURL(redirect) == false) {
+ log.info("Get invalid redirect url, skip it: " + redirect);
+ continue;
+ }
}
}
@@ -112,4 +119,18 @@ public class HadoopStatusGetter {
}
}
+ private static boolean isValidURL(String value) {
+ if (StringUtils.isNotEmpty(value)) {
+ java.net.URL url;
+ try {
+ url = new java.net.URL(value);
+ } catch (MalformedURLException var5) {
+ return false;
+ }
+
+ return StringUtils.isNotEmpty(url.getProtocol()) && StringUtils.isNotEmpty(url.getHost());
+ }
+
+ return false;
+ }
}
[2/2] incubator-kylin git commit: update bin/metastore.sh to use
bin/kylin.sh
Posted by sh...@apache.org.
update bin/metastore.sh to use bin/kylin.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1afb32dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1afb32dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1afb32dc
Branch: refs/heads/1.x-staging
Commit: 1afb32dc55558562d3b096a77bcb0faf7e5495a0
Parents: db7f8be
Author: shaofengshi <sh...@apache.org>
Authored: Mon Nov 16 16:30:04 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Nov 16 16:30:04 2015 +0800
----------------------------------------------------------------------
bin/metastore.sh | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1afb32dc/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
index 53aaa80..39593d4 100755
--- a/bin/metastore.sh
+++ b/bin/metastore.sh
@@ -28,9 +28,6 @@
dir=$(dirname ${0})
source ${dir}/check-env.sh
-_jobjar=`ls ${KYLIN_HOME}/lib |grep kylin-job`
-_fulljobjar="${KYLIN_HOME}/lib/${_jobjar}"
-
if [ $1 == "backup" ]
then
@@ -41,7 +38,7 @@ then
echo "Starting backup to ${_file}"
mkdir -p ${_file}
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool download ${_file}
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool download ${_file}
echo "metadata store backed up to ${_file}"
elif [ $1 == "restore" ]
@@ -49,17 +46,17 @@ then
_file=$2
echo "Starting restoring $_file"
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool upload $_file
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool upload $_file
elif [ $1 == "reset" ]
then
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.common.persistence.ResourceTool reset
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool reset
elif [ $1 == "clean" ]
then
- hbase org.apache.hadoop.util.RunJar ${_fulljobjar} org.apache.kylin.job.hadoop.cube.MetadataCleanupJob "${@:2}"
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.job.hadoop.cube.MetadataCleanupJob "${@:2}"
else
echo "usage: metastore.sh backup"