You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/03 08:32:29 UTC
[1/2] incubator-kylin git commit: KYLIN-942 fix bug in DebugTomcat
pom and Backdoor toggles
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 101ef86ac -> b62e73b76
KYLIN-942 fix bug in DebugTomcat pom and Backdoor toggles
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8970f221
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8970f221
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8970f221
Branch: refs/heads/2.x-staging
Commit: 8970f221816de85378d88653df769c68e7e4cae7
Parents: 101ef86
Author: honma <ho...@ebay.com>
Authored: Tue Nov 3 11:34:43 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Nov 3 11:34:43 2015 +0800
----------------------------------------------------------------------
server/pom.xml | 4 ++++
.../kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 9 ++-------
2 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8970f221/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 14a90d3..d51638f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -103,6 +103,10 @@
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8970f221/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 77a5db5..6ad30e2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -161,6 +161,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
final AtomicInteger totalScannedCount = new AtomicInteger(0);
+ final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior();
+ logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
for (int i = 0; i < rawScans.size(); ++i) {
final int shardIndex = i;
@@ -176,13 +178,6 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
builder.addHbaseColumnsToGT(intList);
}
- // debug/profiling purpose
- String toggle = BackdoorToggles.getCoprocessorBehavior();
- if (toggle == null) {
- toggle = CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
- } else {
- logger.info("The execution of this query will use " + toggle + " as endpoint's behavior");
- }
builder.setBehavior(toggle);
Collection<CubeVisitProtos.CubeVisitResponse> results;
[2/2] incubator-kylin git commit: KYLIN-1119 fix
find_hive_dependency.sh
Posted by ma...@apache.org.
KYLIN-1119 fix find_hive_dependency.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b62e73b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b62e73b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b62e73b7
Branch: refs/heads/2.x-staging
Commit: b62e73b76a41dfa56f5f7fb8603dadc221827f75
Parents: 8970f22
Author: honma <ho...@ebay.com>
Authored: Tue Nov 3 15:37:02 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Nov 3 15:37:02 2015 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 17 ++++++++++++++---
.../kylin/engine/mr/common/AbstractHadoopJob.java | 2 +-
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b62e73b7/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index 98d0860..4982c93 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -19,10 +19,19 @@ do
hive_conf_path=$data
fi
done
-hive_home=`echo $hive_exec_path | awk -F '/lib/' '{print $1}'`/
-hcatalog=`find $hive_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
-hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
+# in some versions of hdp hcatalog is not in hive's classpath, find it separately
+if [ -z "$HCAT_HOME" ]
+then
+ echo "HCAT_HOME not found, try to find hcatalog path from hdp home"
+ hdp_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+ hcatalog_home=${hdp_home}/hive-hcatalog
+else
+ echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
+ hcatalog_home=${HCAT_HOME}
+fi
+
+hcatalog=`find ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
if [ -z "$hcatalog" ]
then
@@ -30,6 +39,8 @@ then
exit 1
fi
+
+hive_lib=`find "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
echo "hive dependency: $hive_dependency"
export hive_dependency
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b62e73b7/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 1455664..ffa7c13 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -158,7 +158,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hbase.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
Configuration jobConf = job.getConfiguration();
String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);