You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/21 03:43:08 UTC
[01/30] kylin git commit: minor,
enhance project version search pattern [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase0.98 3e9f620d5 -> a5ec512be (forced update)
minor, enhance project version search pattern
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d8bf2b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d8bf2b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d8bf2b3
Branch: refs/heads/master-hbase0.98
Commit: 9d8bf2b340738289d0a3d23664868ae3f5b40cd1
Parents: 3a3e967
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 13 18:57:01 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 13 18:57:01 2017 +0800
----------------------------------------------------------------------
build/script/compress.sh | 2 +-
build/script/download-tomcat.sh | 2 +-
build/script/package.sh | 2 +-
build/script/package_diag.sh | 4 ++--
build/script/package_libs.sh | 4 ++--
build/script/prepare.sh | 2 +-
build/script/prepare_libs.sh | 4 ++--
7 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/compress.sh
----------------------------------------------------------------------
diff --git a/build/script/compress.sh b/build/script/compress.sh
index 39e429c..13580f5 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -24,7 +24,7 @@ cd ${dir}/../..
if [ -z "$version" ]
then
echo 'version not set'
- version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
echo "${version}"
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index b3aa509..964400f 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -59,7 +59,7 @@ echo "context.xml overwritten..."
if [ -z "$version" ]
then
echo 'version not set'
- version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
fi
echo "version ${version}"
export version
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/package.sh
----------------------------------------------------------------------
diff --git a/build/script/package.sh b/build/script/package.sh
index a58a5ce..dedd10d 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -50,7 +50,7 @@ fi
dir=$(dirname ${0})
cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev '(^\[|Download\w+:)'`
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
echo "kylin version: ${version}"
export version
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/package_diag.sh
----------------------------------------------------------------------
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
index d0542ab..5d163e3 100755
--- a/build/script/package_diag.sh
+++ b/build/script/package_diag.sh
@@ -39,7 +39,7 @@ fi
dir=$(dirname ${0})
cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
if [ "$version" == "" ];then
echo "Failed to identify kylin version (current: `pwd`)"
@@ -74,4 +74,4 @@ mkdir -p ../dist
tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
rm -rf ${package_name}
-echo "Library package ready: dist/${package_name}.tar.gz"
\ No newline at end of file
+echo "Library package ready: dist/${package_name}.tar.gz"
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/package_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/package_libs.sh b/build/script/package_libs.sh
index 71c1e8c..7ef6b5d 100755
--- a/build/script/package_libs.sh
+++ b/build/script/package_libs.sh
@@ -39,7 +39,7 @@ fi
dir=$(dirname ${0})
cd ${dir}/../..
-version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
if [ "$version" == "" ];then
echo "Failed to identify kylin version (current: `pwd`)"
@@ -97,4 +97,4 @@ mkdir -p ../dist
tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
rm -rf ${package_name}
-echo "Library package ready: dist/${package_name}.tar.gz"
\ No newline at end of file
+echo "Library package ready: dist/${package_name}.tar.gz"
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/prepare.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare.sh b/build/script/prepare.sh
index a1c02ba..a8dde10 100755
--- a/build/script/prepare.sh
+++ b/build/script/prepare.sh
@@ -23,7 +23,7 @@ cd ${dir}/../..
if [ -z "$version" ]
then
echo 'version not set'
- version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
fi
echo "version ${version}"
export version
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d8bf2b3/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index 3dbdf21..ae5c5e4 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -23,7 +23,7 @@ cd ${dir}/../..
if [ -z "$version" ]
then
echo 'version not set'
- version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\['`
+ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -E '^[0-9]+\.[0-9]+\.[0-9]+' `
fi
echo "version ${version}"
@@ -39,4 +39,4 @@ cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/k
chmod 644 build/lib/kylin-job-${version}.jar
chmod 644 build/lib/kylin-coprocessor-${version}.jar
chmod 644 build/lib/kylin-jdbc-${version}.jar
-chmod 644 build/tool/kylin-tool-${version}.jar
\ No newline at end of file
+chmod 644 build/tool/kylin-tool-${version}.jar
[30/30] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5ec512b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5ec512b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5ec512b
Branch: refs/heads/master-hbase0.98
Commit: a5ec512be56e15cc955dad09af0ef9613e61f16d
Parents: f72a3f6
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 21 11:42:23 2017 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 ++++
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 ----
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +++++------------
examples/test_case_data/sandbox/hdfs-site.xml | 84 +++++-------
examples/test_case_data/sandbox/hive-site.xml | 89 +++++--------
examples/test_case_data/sandbox/mapred-site.xml | 57 +++------
examples/test_case_data/sandbox/yarn-site.xml | 127 +++----------------
.../kylin/provision/BuildCubeWithEngine.java | 17 +--
pom.xml | 117 +----------------
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++++++++++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java | 44 +++----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++--
.../storage/hbase/cube/SimpleHBaseStore.java | 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java | 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java | 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml | 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 16 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 595 insertions(+), 870 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 0000000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
deleted file mode 100755
index 2a3d24b..0000000
--- a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index e042aa5..7cb985c 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -47,6 +47,16 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.acl_administer_queue</name>
<value>*</value>
</property>
@@ -57,6 +67,11 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+ <value></value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
<value>*</value>
</property>
@@ -96,4 +111,4 @@
<value>default</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index a4ad5c6..0c5f62b 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -19,6 +19,7 @@
<property>
<name>fs.defaultFS</name>
<value>hdfs://sandbox.hortonworks.com:8020</value>
+ <final>true</final>
</property>
<property>
@@ -38,7 +39,7 @@
<property>
<name>hadoop.proxyuser.falcon.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -48,7 +49,7 @@
<property>
<name>hadoop.proxyuser.hbase.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -67,23 +68,13 @@
</property>
<property>
- <name>hadoop.proxyuser.hdfs.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hdfs.hosts</name>
- <value>*</value>
- </property>
-
- <property>
<name>hadoop.proxyuser.hive.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <value>*</value>
</property>
<property>
@@ -132,15 +123,8 @@
</property>
<property>
- <name>hadoop.security.key.provider.path</name>
- <value></value>
- </property>
-
- <property>
<name>io.compression.codecs</name>
- <value>
- org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec
- </value>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 568de2e..46d5345 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -22,33 +22,8 @@
</property>
<property>
- <name>hbase.bucketcache.ioengine</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.percentage.in.combinedcache</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.size</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bulkload.staging.dir</name>
- <value>/apps/hbase/staging</value>
- </property>
-
- <property>
<name>hbase.client.keyvalue.maxsize</name>
- <value>1048576</value>
- </property>
-
- <property>
- <name>hbase.client.retries.number</name>
- <value>35</value>
+ <value>10485760</value>
</property>
<property>
@@ -63,19 +38,12 @@
<property>
<name>hbase.coprocessor.master.classes</name>
- <value>org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor</value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>
- org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor
- </value>
- </property>
-
- <property>
- <name>hbase.coprocessor.regionserver.classes</name>
- <value></value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
@@ -119,11 +87,6 @@
</property>
<property>
- <name>hbase.hstore.compaction.max</name>
- <value>10</value>
- </property>
-
- <property>
<name>hbase.hstore.compactionThreshold</name>
<value>3</value>
</property>
@@ -140,42 +103,32 @@
<property>
<name>hbase.master.info.port</name>
- <value>16010</value>
+ <value>60010</value>
</property>
<property>
<name>hbase.master.port</name>
- <value>16000</value>
+ <value>60000</value>
</property>
<property>
- <name>hbase.region.server.rpc.scheduler.factory.class</name>
- <value></value>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
</property>
<property>
- <name>hbase.regionserver.global.memstore.size</name>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
<value>0.4</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>30</value>
+ <value>60</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
- <value>16030</value>
- </property>
-
- <property>
- <name>hbase.regionserver.port</name>
- <value>16020</value>
- </property>
-
- <property>
- <name>hbase.regionserver.wal.codec</name>
- <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value>
+ <value>60030</value>
</property>
<property>
@@ -184,26 +137,11 @@
</property>
<property>
- <name>hbase.rpc.controllerfactory.class</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.rpc.engine</name>
- <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
- </property>
-
- <property>
<name>hbase.rpc.protection</name>
<value>PRIVACY</value>
</property>
<property>
- <name>hbase.rpc.timeout</name>
- <value>90000</value>
- </property>
-
- <property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
@@ -220,7 +158,7 @@
<property>
<name>hbase.tmp.dir</name>
- <value>/tmp/hbase-${user.name}</value>
+ <value>/hadoop/hbase</value>
</property>
<property>
@@ -240,27 +178,34 @@
<property>
<name>hfile.block.cache.size</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>phoenix.functions.allowUserDefinedFunctions</name>
- <value></value>
- </property>
-
- <property>
- <name>phoenix.query.timeoutMs</name>
- <value>60000</value>
+ <value>0.40</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
- <value>60000</value>
+ <value>30000</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
-
-</configuration>
\ No newline at end of file
+ <property>
+ <name>hbase.client.pause</name>
+ <value>100</value>
+ <description>General client pause value. Used mostly as value to wait
+ before running a retry of a failed get, region lookup, etc.
+ See hbase.client.retries.number for description of how we backoff from
+ this initial pause amount and how this pause works w/ retries.</description>
+ </property>
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>5</value>
+ <description>Maximum retries. Used as maximum for all retryable
+ operations such as the getting of a cell's value, starting a row update,
+ etc. Retry interval is a rough function based on hbase.client.pause. At
+ first we retry at this interval but then with backoff, we pretty quickly reach
+ retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
+ ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1d9040a..1175fff 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -18,7 +18,12 @@
<property>
<name>dfs.block.access.token.enable</name>
- <value>true</value>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.block.size</name>
+ <value>34217472</value>
</property>
<property>
@@ -42,21 +47,11 @@
</property>
<property>
- <name>dfs.client.retry.policy.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.cluster.administrators</name>
<value>hdfs</value>
</property>
<property>
- <name>dfs.content-summary.limit</name>
- <value>5000</value>
- </property>
-
- <property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:50010</value>
</property>
@@ -69,6 +64,7 @@
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/hdfs/data</value>
+ <final>true</final>
</property>
<property>
@@ -84,6 +80,7 @@
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
+ <final>true</final>
</property>
<property>
@@ -107,18 +104,13 @@
</property>
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.encrypt.data.transfer.cipher.suites</name>
- <value>AES/CTR/NoPadding</value>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>1024</value>
</property>
<property>
- <name>dfs.encryption.key.provider.uri</name>
- <value></value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
<property>
@@ -158,12 +150,7 @@
<property>
<name>dfs.namenode.accesstime.precision</name>
- <value>0</value>
- </property>
-
- <property>
- <name>dfs.namenode.audit.log.async</name>
- <value>true</value>
+ <value>3600000</value>
</property>
<property>
@@ -197,11 +184,6 @@
</property>
<property>
- <name>dfs.namenode.fslock.fair</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
@@ -209,6 +191,7 @@
<property>
<name>dfs.namenode.http-address</name>
<value>sandbox.hortonworks.com:50070</value>
+ <final>true</final>
</property>
<property>
@@ -217,13 +200,9 @@
</property>
<property>
- <name>dfs.namenode.inode.attributes.provider.class</name>
- <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
- </property>
-
- <property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/hdfs/namenode</value>
+ <final>true</final>
</property>
<property>
@@ -232,13 +211,8 @@
</property>
<property>
- <name>dfs.namenode.rpc-address</name>
- <value>sandbox.hortonworks.com:8020</value>
- </property>
-
- <property>
<name>dfs.namenode.safemode.threshold-pct</name>
- <value>0.999</value>
+ <value>1.0f</value>
</property>
<property>
@@ -262,6 +236,16 @@
</property>
<property>
+ <name>dfs.nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs3.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+ <property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
@@ -273,7 +257,7 @@
<property>
<name>dfs.replication</name>
- <value>3</value>
+ <value>1</value>
</property>
<property>
@@ -284,11 +268,13 @@
<property>
<name>dfs.support.append</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
@@ -296,14 +282,4 @@
<value>022</value>
</property>
- <property>
- <name>nfs.exports.allowed.hosts</name>
- <value>* rw</value>
- </property>
-
- <property>
- <name>nfs.file.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
- </property>
-
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index a8c210e..1e78107 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -22,46 +22,11 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>Sandbox</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.maxThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.minThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.synchronous</name>
- <value>true</value>
- </property>
-
- <property>
- <name>atlas.rest.address</name>
- <value>http://sandbox.hortonworks.com:21000</value>
- </property>
-
- <property>
- <name>datanucleus.autoCreateSchema</name>
- <value>false</value>
- </property>
-
- <property>
<name>datanucleus.cache.level2.type</name>
<value>none</value>
</property>
<property>
- <name>datanucleus.fixedDatastore</name>
- <value>true</value>
- </property>
-
- <property>
<name>hive.auto.convert.join</name>
<value>true</value>
</property>
@@ -73,7 +38,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>357913941</value>
+ <value>1000000000</value>
</property>
<property>
@@ -162,16 +127,6 @@
</property>
<property>
- <name>hive.default.fileformat</name>
- <value>TextFile</value>
- </property>
-
- <property>
- <name>hive.default.fileformat.managed</name>
- <value>TextFile</value>
- </property>
-
- <property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>
@@ -207,6 +162,11 @@
</property>
<property>
+ <name>hive.exec.failure.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.max.created.files</name>
<value>100000</value>
</property>
@@ -237,11 +197,6 @@
</property>
<property>
- <name>hive.exec.orc.encoding.strategy</name>
- <value>SPEED</value>
- </property>
-
- <property>
<name>hive.exec.parallel</name>
<value>false</value>
</property>
@@ -252,6 +207,16 @@
</property>
<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>67108864</value>
</property>
@@ -297,6 +262,11 @@
</property>
<property>
+ <name>hive.heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
<name>hive.limit.optimize.enable</name>
<value>true</value>
</property>
@@ -508,7 +478,7 @@
<property>
<name>hive.prewarm.numcontainers</name>
- <value>3</value>
+ <value>10</value>
</property>
<property>
@@ -518,7 +488,7 @@
<property>
<name>hive.security.authorization.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -538,7 +508,7 @@
<property>
<name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
</property>
<property>
@@ -563,7 +533,12 @@
<property>
<name>hive.server2.enable.doAs</name>
- <value>false</value>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.impersonation</name>
+ <value>true</value>
</property>
<property>
@@ -573,7 +548,7 @@
<property>
<name>hive.server2.logging.operation.log.location</name>
- <value>/tmp/hive/operation_logs</value>
+ <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
</property>
<property>
@@ -678,7 +653,7 @@
<property>
<name>hive.tez.container.size</name>
- <value>1024</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index be470f9..e90f594 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>64</value>
+ <value>128</value>
</property>
<property>
@@ -27,13 +27,13 @@
</property>
<property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.map.memory.mb</name>
+ <value>512</value>
</property>
<property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>512</value>
</property>
<property>
@@ -48,9 +48,7 @@
<property>
<name>mapreduce.admin.user.env</name>
- <value>
- LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64
- </value>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
</property>
<property>
@@ -60,9 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>
- $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure
- </value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,18 +77,14 @@
</property>
<property>
- <name>mapreduce.job.counters.max</name>
- <value>130</value>
- </property>
-
- <property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
</property>
+ <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
+ <value>1</value>
</property>
<property>
@@ -116,28 +108,13 @@
</property>
<property>
- <name>mapreduce.jobhistory.recovery.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.class</name>
- <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
- <value>/hadoop/mapreduce/jhs</value>
- </property>
-
- <property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>sandbox.hortonworks.com:19888</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx1228m</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -147,7 +124,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>1536</value>
+ <value>512</value>
</property>
<property>
@@ -182,7 +159,7 @@
<property>
<name>mapreduce.reduce.java.opts</name>
- <value>-Xmx1638m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -192,7 +169,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>2048</value>
+ <value>512</value>
</property>
<property>
@@ -242,7 +219,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>859</value>
+ <value>128</value>
</property>
<property>
@@ -257,7 +234,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx819m -Dhdp.version=${hdp.version}</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -267,7 +244,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>1024</value>
+ <value>512</value>
</property>
<property>
@@ -275,4 +252,4 @@
<value>/user</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index ebdf44a..8256158 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -18,7 +18,7 @@
<property>
<name>hadoop.registry.rm.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -28,29 +28,22 @@
<property>
<name>yarn.acl.enable</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>yarn.admin.acl</name>
- <value>*</value>
+ <value></value>
</property>
<property>
<name>yarn.application.classpath</name>
- <value>
- $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*
- </value>
- </property>
-
- <property>
- <name>yarn.authorization-provider</name>
- <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
</property>
<property>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -79,11 +72,6 @@
</property>
<property>
- <name>yarn.node-labels.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.node-labels.fs-store.retry-policy-spec</name>
<value>2000, 500</value>
</property>
@@ -94,6 +82,11 @@
</property>
<property>
+ <name>yarn.node-labels.manager-class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.address</name>
<value>0.0.0.0:45454</value>
</property>
@@ -105,7 +98,7 @@
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle,spark_shuffle</value>
+ <value>mapreduce_shuffle</value>
</property>
<property>
@@ -114,11 +107,6 @@
</property>
<property>
- <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
- <value>org.apache.spark.network.yarn.YarnShuffleService</value>
- </property>
-
- <property>
<name>yarn.nodemanager.bind-host</name>
<value>0.0.0.0</value>
</property>
@@ -160,7 +148,7 @@
<property>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -255,12 +243,12 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
- <value>7168</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>80</value>
+ <value>100</value>
</property>
<property>
@@ -349,11 +337,6 @@
</property>
<property>
- <name>yarn.resourcemanager.scheduler.monitor.enable</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.resourcemanager.state-store.max-completed-applications</name>
<value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
@@ -385,7 +368,7 @@
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8090</value>
+ <value>localhost:8090</value>
</property>
<property>
@@ -425,7 +408,7 @@
<property>
<name>yarn.resourcemanager.zk-address</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <value>localhost:2181</value>
</property>
<property>
@@ -450,22 +433,12 @@
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
- <value>7168</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-vcores</name>
- <value>3</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-vcores</name>
- <value>1</value>
+ <value>1536</value>
</property>
<property>
@@ -494,41 +467,6 @@
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
- <value>/ats/active/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
- <value>3600</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
- <value>/ats/done/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
- <value>org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
- <value>60</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
- <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.generic-application-history.store-class</name>
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
@@ -544,11 +482,6 @@
</property>
<property>
- <name>yarn.timeline-service.leveldb-state-store.path</name>
- <value>/hadoop/yarn/timeline</value>
- </property>
-
- <property>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
<value>/hadoop/yarn/timeline</value>
</property>
@@ -574,23 +507,8 @@
</property>
<property>
- <name>yarn.timeline-service.plugin.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.state-store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
@@ -604,11 +522,6 @@
</property>
<property>
- <name>yarn.timeline-service.version</name>
- <value>1.5</value>
- </property>
-
- <property>
<name>yarn.timeline-service.webapp.address</name>
<value>sandbox.hortonworks.com:8188</value>
</property>
@@ -618,4 +531,4 @@
<value>sandbox.hortonworks.com:8190</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index df6c45e..d819098 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -32,9 +32,11 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,7 +60,6 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -95,10 +96,10 @@ public class BuildCubeWithEngine {
logger.error("error", e);
exitCode = 1;
}
-
+
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
-
+
System.exit(exitCode);
}
@@ -359,10 +360,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
- String tableName = segment.getStorageLocationIdentifier();
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ String tableName = segment.getStorageLocationIdentifier();
+ try (HTable table = new HTable(conf, tableName)) {
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0e41dcf..3981f07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,15 +46,15 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0</hadoop2.version>
+ <yarn.version>2.6.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>0.14.0</hive.version>
+ <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
<!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
@@ -71,7 +71,7 @@
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.8</zookeeper.version>
- <curator.version>2.7.1</curator.version>
+ <curator.version>2.6.0</curator.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
<jsch.version>0.1.53</jsch.version>
@@ -842,11 +842,6 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
-
- <repository>
- <id>cloudera</id>
- <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
- </repository>
</repositories>
<build>
@@ -1200,106 +1195,6 @@
</build>
</profile>
<profile>
- <id>cdh5.7</id>
- <properties>
- <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
- <yarn.version>2.6.0-cdh5.7.0</yarn.version>
- <hive.version>1.1.0-cdh5.7.0</hive.version>
- <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
- <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
- <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <fork>true</fork>
- <meminitial>1024m</meminitial>
- <maxmem>2048m</maxmem>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-jamm</id>
- <goals>
- <goal>copy</goal>
- </goals>
- <phase>generate-test-resources</phase>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>com.github.jbellis</groupId>
- <artifactId>jamm</artifactId>
- <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
- <destFileName>jamm.jar</destFileName>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <configuration>
- <append>true</append>
- <destFile>
- ${sonar.jacoco.reportPath}
- </destFile>
- </configuration>
- <executions>
- <execution>
- <id>pre-test</id>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- <configuration>
- <propertyName>surefireArgLine</propertyName>
- </configuration>
- </execution>
- <execution>
- <id>post-test</id>
- <phase>test</phase>
- <goals>
- <goal>report</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.19.1</version>
- <configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
- <excludes>
- <exclude>**/IT*.java</exclude>
- </excludes>
- <systemProperties>
- <property>
- <name>buildCubeUsingProvidedData</name>
- <value>false</value>
- </property>
- <property>
- <name>log4j.configuration</name>
- <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
- </property>
- </systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<!-- This profile adds/overrides few features of the 'apache-release'
profile in the parent pom. -->
<id>apache-release</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 8095bf8..ea68855 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- Table getTable(String tableName) throws IOException;
+ HTableInterface getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index cc76b87..d9326f5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private Table mockedAclTable;
- private Table mockedUserTable;
+ private HTableInterface mockedAclTable;
+ private HTableInterface mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index 972eea9..d0aa0ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements Table {
+public class MockHTable implements HTableInterface {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,6 +114,14 @@ public class MockHTable implements Table {
this.columnFamilies.add(columnFamily);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public byte[] getTableName() {
+ return tableName.getBytes();
+ }
+
@Override
public TableName getName() {
return null;
@@ -192,8 +200,8 @@ public class MockHTable implements Table {
}
@Override
- public boolean[] existsAll(List<Get> list) throws IOException {
- return new boolean[0];
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ return new Boolean[0];
}
/**
@@ -298,6 +306,15 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ // FIXME: implement
+ return null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -429,7 +446,7 @@ public class MockHTable implements Table {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow();
+ filter.filterRow(nkvs);
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -518,11 +535,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -543,7 +555,7 @@ public class MockHTable implements Table {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDelete()) {
+ if (kv.isDeleteFamily()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -580,11 +592,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -598,7 +605,7 @@ public class MockHTable implements Table {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, null);
+ return incrementColumnValue(row, family, qualifier, amount, true);
}
@Override
@@ -610,6 +617,37 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
+ if (check(row, family, qualifier, null)) {
+ Put put = new Put(row);
+ put.add(family, qualifier, Bytes.toBytes(amount));
+ put(put);
+ return amount;
+ }
+ long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
+ data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
+ return newValue;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isAutoFlush() {
+ return true;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void flushCommits() throws IOException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void close() throws IOException {
}
@@ -635,6 +673,29 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public void setAutoFlush(boolean autoFlush) {
+ throw new NotImplementedException();
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ throw new NotImplementedException();
+
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ throw new NotImplementedException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index d1a1384..1d520c4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,8 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -59,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(userTableName);
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 3e3efec..d693a67 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- Table htable = null;
+ HTableInterface htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,16 +226,17 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -249,7 +250,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -265,6 +266,7 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -282,7 +284,7 @@ public class AclService implements MutableAclService {
throw e;
}
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -293,16 +295,17 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a6310d1..cb554c0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -27,7 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -393,24 +395,33 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
- Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
+
+ Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+ HTable table = null;
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ try {
+ table = new HTable(hconf, tableName);
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- regionCount = sizeMap.size();
+ for (long s : sizeMap.values()) {
+ tableSize += s;
+ }
+
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 6a38638..8972456 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -48,11 +48,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -163,13 +163,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -195,13 +196,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +215,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- Table htable = null;
+ HTableInterface htable = null;
try {
- org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = conn.getTable(userTableName);
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index ab54882..07c7c6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
-
- put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+ put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,13 +163,14 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -184,7 +185,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -215,7 +216,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- Table htable = null;
+ HTableInterface htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..2351412 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -39,9 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -63,7 +63,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
+ private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -74,7 +74,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (Connection conn : connPool.values()) {
+ for (HConnection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -143,7 +143,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -207,9 +207,9 @@ public class HBaseConnection {
// ============================================================================
- // returned Connection can be shared by multiple threads and does not require close()
+ // returned HConnection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static Connection get(String url) {
+ public static HConnection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -217,13 +217,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- Connection connection = connPool.get(url);
+ HConnection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = ConnectionFactory.createConnection(conf);
+ connection = HConnectionManager.createConnection(conf);
connPool.put(url, connection);
}
@@ -242,8 +242,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static boolean tableExists(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -263,18 +263,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
- Admin hbase = conn.getAdmin();
- TableName tableName = TableName.valueOf(table);
+ public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
+
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(tableName, newFamilyDescriptor(family));
+ hbase.addColumn(table, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -327,8 +327,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static void deleteTable(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
if (!tableExists(conn, tableName)) {
@@ -338,10 +338,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
- hbase.disableTable(TableName.valueOf(tableName));
+ if (hbase.isTableEnabled(tableName)) {
+ hbase.disableTable(tableName);
}
- hbase.deleteTable(TableName.valueOf(tableName));
+ hbase.deleteTable(tableName);
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- Connection getConnection() throws IOException {
+ HConnection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
+ table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
+ table.flushCommits();
+
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
+ table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -314,7 +317,7 @@ public class HBaseResourceStore extends ResourceStore {
}
- private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -333,7 +336,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
@@ -358,7 +361,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -366,8 +369,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.addColumn(B_FAMILY, B_COLUMN, content);
- put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.add(B_FAMILY, B_COLUMN, content);
+ put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
[20/30] kylin git commit: KYLIN-2280 minor,
reset to init before modify ports
Posted by li...@apache.org.
KYLIN-2280 minor,reset to init before modify ports
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40249fee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40249fee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40249fee
Branch: refs/heads/master-hbase0.98
Commit: 40249fee78a8a078d461ea66009111082732d27b
Parents: 6c800d6
Author: xiefan46 <95...@qq.com>
Authored: Fri Mar 17 11:27:08 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 17 12:53:11 2017 +0800
----------------------------------------------------------------------
build/bin/kylin_port_replace_util.sh | 15 ++++++++++-----
build/script/download-tomcat.sh | 1 +
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/40249fee/build/bin/kylin_port_replace_util.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin_port_replace_util.sh b/build/bin/kylin_port_replace_util.sh
index a51e60d..366d8d0 100755
--- a/build/bin/kylin_port_replace_util.sh
+++ b/build/bin/kylin_port_replace_util.sh
@@ -31,6 +31,7 @@ else
fi
#variables
+TOMCAT_INIT_FILE="${KYLIN_HOME}/tomcat/conf/server.xml.init"
TOMCAT_BACKUP_FILE="${KYLIN_HOME}/tomcat/conf/server.xml.backup"
TOMCAT_CONFIG_FILE="${KYLIN_HOME}/tomcat/conf/server.xml"
KYLIN_CONFIG_FILE="${KYLIN_HOME}/conf/kylin.properties"
@@ -49,6 +50,15 @@ then
exit 1
fi
+
+ #backup tomccat file
+ if [ ! -f ${TOMCAT_BACKUP_FILE} ]; then
+ cp -f ${TOMCAT_CONFIG_FILE} ${TOMCAT_BACKUP_FILE}
+ fi
+
+ #force reset
+ cp -f ${TOMCAT_INIT_FILE} ${TOMCAT_CONFIG_FILE} #reset if exist
+
#back or reset
if [ ! -f ${KYLIN_BACKUP_FILE} ]; then #backup if not exist
cp -f ${KYLIN_CONFIG_FILE} ${KYLIN_BACKUP_FILE}
@@ -56,11 +66,6 @@ then
cp -r ${KYLIN_BACKUP_FILE} ${KYLIN_CONFIG_FILE} #reset if exist
fi
- if [ ! -f ${TOMCAT_BACKUP_FILE} ]; then #backup if not exist
- cp -f ${TOMCAT_CONFIG_FILE} ${TOMCAT_BACKUP_FILE}
- else
- cp -r ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE} #reset if exist
- fi
#replace ports in kylin.properties
new_kylin_port=`expr ${KYLIN_DEFAULT_PORT} + ${OFFSET}`
http://git-wip-us.apache.org/repos/asf/kylin/blob/40249fee/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index 964400f..b0ff137 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -51,6 +51,7 @@ rm -rf build/tomcat/webapps/*
mv build/tomcat/conf/server.xml build/tomcat/conf/server.xml.bak
mv build/tomcat/conf/context.xml build/tomcat/conf/context.xml.bak
cp build/deploy/server.xml build/tomcat/conf/server.xml
+cp build/deploy/server.xml build/tomcat/conf/server.xml.init
echo "server.xml overwritten..."
cp build/deploy/context.xml build/tomcat/conf/context.xml
echo "context.xml overwritten..."
[10/30] kylin git commit: refine metastore.sh refresh-cube-signature
Posted by li...@apache.org.
refine metastore.sh refresh-cube-signature
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/242b3317
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/242b3317
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/242b3317
Branch: refs/heads/master-hbase0.98
Commit: 242b331746f6e9b56da5d5b1ed13b15cbbeff342
Parents: 7bc8aaf
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Mar 15 13:53:35 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 13:53:35 2017 +0800
----------------------------------------------------------------------
build/bin/metastore.sh | 1 +
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 +-
.../src/main/java/org/apache/kylin/rest/service/JobService.java | 2 +-
3 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/242b3317/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index cbce279..23464e6 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -101,6 +101,7 @@ else
echo "usage: metastore.sh backup"
echo " metastore.sh fetch DATA"
echo " metastore.sh reset"
+ echo " metastore.sh refresh-cube-signature"
echo " metastore.sh restore PATH_TO_LOCAL_META"
echo " metastore.sh list RESOURCE_PATH"
echo " metastore.sh cat RESOURCE_PATH"
http://git-wip-us.apache.org/repos/asf/kylin/blob/242b3317/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a4f7c76..ba77f06 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -345,7 +345,7 @@ public class CubeService extends BasicService {
throw new JobException("Enable is not allowed with a running job.");
}
if (!cube.getDescriptor().checkSignature()) {
- throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures");
+ throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures, then reload metadata to take effect");
}
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/242b3317/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 3f17a81..1fafb12 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -318,7 +318,7 @@ public class JobService extends BasicService implements InitializingBean {
private void checkCubeDescSignature(CubeInstance cube) {
if (!cube.getDescriptor().checkSignature())
- throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures");
+ throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures, then reload metadata to take effect");
}
public JobInstance getJobInstance(String uuid) throws IOException, JobException {
[09/30] kylin git commit: minor, add mr tasks in job diag
Posted by li...@apache.org.
minor, add mr tasks in job diag
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7bc8aafd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7bc8aafd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7bc8aafd
Branch: refs/heads/master-hbase0.98
Commit: 7bc8aafd124309d3ebfa4aca89701bd20cd0d589
Parents: d4768c1
Author: lidongsjtu <li...@apache.org>
Authored: Wed Mar 15 00:23:20 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Mar 15 00:24:21 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/tool/MrJobInfoExtractor.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7bc8aafd/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index 55b54a5..483694b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -150,6 +150,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
extractTaskCounters(exportDir, jobUrlPrefix);
}
extractJobCounters(exportDir, jobUrlPrefix);
+ extractJobTasks(exportDir, jobUrlPrefix);
extractJobConf(exportDir, jobUrlPrefix);
} catch (Exception e) {
logger.warn("Failed to get mr tasks rest response.", e);
@@ -160,14 +161,22 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
String url = jobUrlPrefix + "/counters";
String response = getHttpResponse(url);
try {
- File counterDir = new File(exportDir, "counters");
- FileUtils.forceMkdir(counterDir);
FileUtils.writeStringToFile(new File(exportDir, "job_counters.json"), response, Charset.defaultCharset());
} catch (Exception e) {
logger.warn("Failed to get mr counters rest response.", e);
}
}
+ private void extractJobTasks(File exportDir, String jobUrlPrefix) {
+ String url = jobUrlPrefix + "/tasks";
+ String response = getHttpResponse(url);
+ try {
+ FileUtils.writeStringToFile(new File(exportDir, "job_tasks.json"), response, Charset.defaultCharset());
+ } catch (Exception e) {
+ logger.warn("Failed to get mr counters rest response.", e);
+ }
+ }
+
private void extractTaskCounters(File exportDir, String jobUrlPrefix) {
try {
String tasksUrl = jobUrlPrefix + "/tasks/";
[29/30] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b12173d..51b21b3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,13 +26,12 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -87,13 +86,14 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final BufferedMutator table;
+ final HTableInterface table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getBufferedMutator(htableName);
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getTable(htableName);
+ table.setAutoFlush(false, true);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.mutate(put);
+ table.put(put);
}
@Override
public void close() throws IOException {
- table.flush();
+ table.flushCommits();
table.close();
}
}
class Reader implements IGTScanner {
- final Table table;
+ final HTableInterface table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 82b67b6..1cd6694 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,9 +26,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -52,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +116,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -172,7 +171,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicReference<RuntimeException> regionErrorHolder = new AtomicReference<>();
try {
- Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
+ HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 33f8d90..a24d517 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -25,12 +25,11 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -132,8 +131,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index cde127e..1e550f9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -195,7 +195,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -234,7 +234,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = (HRegion)env.getRegion();
+ region = env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index feb4842..2814ad6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,8 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -80,8 +79,7 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin admin = conn.getAdmin();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -94,7 +92,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(TableName.valueOf(tableName))) {
+ if (admin.tableExists(tableName)) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -103,7 +101,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -112,7 +110,8 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -127,7 +126,8 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index df3cf08..eacff9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,21 +100,19 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
-
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 6587d4e..d5b36df 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final Table hTable;
+ private final HTableInterface hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,6 +117,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
+ hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 2f7e164..5b2441c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,20 +69,19 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..a150607 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,11 +21,9 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -40,8 +38,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -52,7 +50,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
+ hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 581de38..68c0a39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -26,19 +26,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -89,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static Admin hbaseAdmin;
+ private static HBaseAdmin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -233,7 +233,6 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
-
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -327,8 +326,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -450,11 +449,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -474,6 +473,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -504,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -539,12 +539,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +562,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config) {
+ private static void updateMeta(KylinConfig config){
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 20d0f7d..8bd4abf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,8 +130,9 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
+
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -188,10 +189,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.disableTable(sepNameList[0]);
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
- hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.modifyTable(sepNameList[0], desc);
+ hbaseAdmin.enableTable(sepNameList[0]);
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..c9e969f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,8 +44,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -86,8 +85,7 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -205,10 +203,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -229,7 +227,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -244,7 +242,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -270,15 +268,16 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.modifyTable(tableName, desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.enableTable(tableName);
return true;
}
- private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+
+ private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -299,12 +298,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final Admin hbaseAdmin;
+ private final HBaseAdmin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -425,7 +424,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 1cdb2f8..61c73d5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -236,9 +235,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -258,6 +257,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index dd5f8fa..86ba22f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(Connection conn) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void prepareData(HConnection conn) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..6749d6c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,11 +24,9 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -57,8 +55,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -73,12 +71,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 1daca0a..937b65f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -32,15 +31,12 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -62,31 +58,30 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ public HBaseRegionSizeCalculator(HTable table) throws IOException {
+ this(table, new HBaseAdmin(table.getConfiguration()));
+ }
- Table table = null;
- Admin admin = null;
- try {
- table = hbaseConnection.getTable(TableName.valueOf(tableName));
- admin = hbaseConnection.getAdmin();
+ /** Constructor for unit testing */
+ HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ try {
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
+ logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
// Get regions for table.
- RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
- List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
+ Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionLocation hRegionLocation : regionLocationList) {
- tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
+ for (HRegionInfo regionInfo : tableRegionInfos) {
+ tableRegions.add(regionInfo.getRegionName());
}
- ClusterStatus clusterStatus = admin.getClusterStatus();
+ ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -110,7 +105,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(admin);
+ IOUtils.closeQuietly(hBaseAdmin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..266f7e7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,10 +23,9 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -43,8 +42,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index 8dd2164..1db60fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- Connection conn = getConnection();
- Admin hadmin = conn.getAdmin();
+ HConnection conn = getConnection();
+ HBaseAdmin hadmin = new HBaseAdmin(conn);
try {
- boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
+ boolean tableExist = hadmin.tableExists(tableName);
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -120,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
byte[] key = new byte[8 + 4];//time + id
@@ -136,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.addColumn(CF, QN, cell);
+ put.add(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -172,8 +172,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -208,7 +208,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(Table table) throws IOException {
+ private static long getFirstKeyTime(HTableInterface table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -226,8 +226,8 @@ public class HbaseStreamingInput {
}
- private static Connection getConnection() throws IOException {
- return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ private static HConnection getConnection() throws IOException {
+ return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ea05ab2..ca1a060 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,11 +23,10 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -51,8 +50,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..8ff5b0f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,14 +30,10 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +52,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+
// get all kylin hbase tables
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -77,13 +73,12 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- TableName tableName = TableName.valueOf(htableName);
- if (hbaseAdmin.tableExists(tableName)) {
- if (hbaseAdmin.isTableEnabled(tableName)) {
- hbaseAdmin.disableTable(tableName);
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(tableName);
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index bba6745..1ea8e8d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,13 +22,12 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -60,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- Connection conn = null;
- Table table = null;
+ HConnection conn = null;
+ HTableInterface table = null;
ResultScanner scanner = null;
try {
- conn = ConnectionFactory.createConnection(hconf);
- table = conn.getTable(TableName.valueOf(hbaseTable));
+ conn = HConnectionManager.createConnection(hconf);
+ table = conn.getTable(hbaseTable);
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index db516bb..01edb1f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,12 +22,11 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -71,8 +70,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- Connection conn = ConnectionFactory.createConnection(conf);
- Table tableInterface = conn.getTable(TableName.valueOf(htableName));
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..23e7e10 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,9 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -59,7 +57,6 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,8 +77,7 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -157,22 +153,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- Admin hbaseAdmin;
+ HBaseAdmin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 42a54c8..e36f662 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,18 +24,16 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,15 +49,14 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
- hbaseAdmin = conn.getAdmin();
+ this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -169,9 +166,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.enableTable(tableName);
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 278c2b8..072ee44 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -60,16 +60,6 @@
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <scope>provided</scope>
- </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c162a76..8ada323 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -462,11 +462,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -486,6 +486,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -551,12 +552,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5ec512b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[13/30] kylin git commit: minor, more verbose on job diag
Posted by li...@apache.org.
minor, more verbose on job diag
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5155994e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5155994e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5155994e
Branch: refs/heads/master-hbase0.98
Commit: 5155994e1a84e47d9ce8d90d3787a28910673b66
Parents: e4ed232
Author: lidongsjtu <li...@apache.org>
Authored: Wed Mar 15 14:44:17 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Mar 15 14:51:55 2017 +0800
----------------------------------------------------------------------
build/bin/diag.sh | 4 +-
build/conf/kylin-tools-log4j.properties | 2 +-
.../apache/kylin/tool/MrJobInfoExtractor.java | 208 +++++++++++++------
3 files changed, 149 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5155994e/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index a995774..bb1bdc4 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -52,14 +52,14 @@ then
if [ ${#patient} -eq 36 ]; then
hbase ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=kylin-tool-log4j.properties \
+ -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties \
-Dcatalina.home=${tomcat_root} \
org.apache.kylin.tool.JobDiagnosisInfoCLI \
-jobId $patient \
-destDir $destDir || exit 1
else
hbase ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=kylin-server-log4j.properties \
+ -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties \
-Dcatalina.home=${tomcat_root} \
org.apache.kylin.tool.DiagnosisInfoCLI \
-project -all \
http://git-wip-us.apache.org/repos/asf/kylin/blob/5155994e/build/conf/kylin-tools-log4j.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 2ccd772..d47f9a2 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -35,4 +35,4 @@ log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %
#log4j.logger.org.apache.hadoop=ERROR
log4j.logger.org.apache.kylin=DEBUG
log4j.logger.org.springframework=WARN
-log4j.logger.org.apache.commons.httpclient=WARN
+log4j.logger.org.apache.kylin.tool.shaded=INFO
http://git-wip-us.apache.org/repos/asf/kylin/blob/5155994e/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index 483694b..b9bf2de 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -21,6 +21,7 @@ package org.apache.kylin.tool;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -45,12 +46,13 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
public class MrJobInfoExtractor extends AbstractInfoExtractor {
private static final Logger logger = LoggerFactory.getLogger(MrJobInfoExtractor.class);
@SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_COUNTERS = OptionBuilder.withArgName("includeCounters").hasArg().isRequired(false).withDescription("Specify whether to include mr task counters to extract. Default true.").create("includeCounters");
+ private static final Option OPTION_INCLUDE_DETAILS = OptionBuilder.withArgName("includeTasks").hasArg().isRequired(false).withDescription("Specify whether to include mr task details to extract. Default true.").create("includeTasks");
@SuppressWarnings("static-access")
private static final Option OPTION_MR_JOB_ID = OptionBuilder.withArgName("mrJobId").hasArg().isRequired(false).withDescription("Specify MR Job Id").create("mrJobId");
@@ -60,7 +62,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
public MrJobInfoExtractor() {
packageType = "MR";
- options.addOption(OPTION_INCLUDE_COUNTERS);
+ options.addOption(OPTION_INCLUDE_DETAILS);
options.addOption(OPTION_MR_JOB_ID);
}
@@ -79,7 +81,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
return m.group(1) + m.group(2) + ":19888";
}
}
- logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set read from hadoop configuration");
+ logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");
Configuration conf = HadoopUtil.getCurrentConfiguration();
String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
@@ -120,69 +122,113 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
return msg;
}
- private void extractTaskCounter(String taskId, File exportDir, String taskUrl, String id) throws IOException {
+ private void extractTaskDetail(String taskId, String nodeId, String user, File exportDir, String taskUrl, String urlBase) throws IOException {
try {
- String response = getHttpResponse(taskUrl + taskId + "/counters");
- FileUtils.writeStringToFile(new File(exportDir, id + "_" + taskId + ".json"), response, Charset.defaultCharset());
+ if (StringUtils.isEmpty(taskId)) {
+ return;
+ }
+
+ String taskUrlBase = taskUrl + taskId;
+ File destDir = new File(exportDir, taskId);
+
+ // get task basic info
+ String taskInfo = saveHttpResponseQuietly(new File(destDir, "task.json"), taskUrlBase);
+ JsonNode taskAttempt = new ObjectMapper().readTree(taskInfo).path("task").path("successfulAttempt");
+ String succAttemptId = taskAttempt.textValue();
+
+ String attemptInfo = saveHttpResponseQuietly(new File(destDir, "task_attempts.json"), taskUrlBase + "/attempts/" + succAttemptId);
+ JsonNode attemptAttempt = new ObjectMapper().readTree(attemptInfo).path("taskAttempt").path("assignedContainerId");
+ String containerId = attemptAttempt.textValue();
+
+ // save task counters
+ saveHttpResponseQuietly(new File(destDir, "task_counters.json"), taskUrlBase + "/counters");
+
+ // save task logs
+ String logUrl = urlBase + "/jobhistory/logs/" + nodeId + "/" + containerId + "/" + succAttemptId + "/" + user + "/syslog/?start=0";
+ logger.debug("Fetch task log from url: " + logUrl);
+
+ saveHttpResponseQuietly(new File(destDir, "task_log.txt"), logUrl);
} catch (Exception e) {
logger.warn("Failed to get task counters rest response" + e);
}
}
- private void extractJobConf(File exportDir, String jobUrlPrefix) throws IOException {
+ private String saveHttpResponseQuietly(File dest, String url) {
+ String response = null;
+
try {
- String confUrl = jobUrlPrefix + "/conf/";
- String response = getHttpResponse(confUrl);
- FileUtils.writeStringToFile(new File(exportDir, "job_conf.json"), response, Charset.defaultCharset());
+ response = getHttpResponse(url);
+ FileUtils.forceMkdir(dest.getParentFile());
+ FileUtils.writeStringToFile(dest, response, Charset.defaultCharset());
+ return response;
} catch (Exception e) {
- logger.warn("Failed to get job conf rest response.", e);
+ logger.warn("Failed to get http response from {}.", url, e);
}
+
+ return response;
}
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
try {
- boolean includeTaskCounter = optionsHelper.hasOption(OPTION_INCLUDE_COUNTERS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_COUNTERS)) : true;
+ boolean includeTaskDetails = optionsHelper.hasOption(OPTION_INCLUDE_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_DETAILS)) : true;
String mrJobId = optionsHelper.getOptionValue(OPTION_MR_JOB_ID);
- String jobUrlPrefix = getRestCheckUrl() + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
+ String jobUrlBase = getRestCheckUrl();
+ String jobUrlPrefix = jobUrlBase + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
- if (includeTaskCounter) {
- extractTaskCounters(exportDir, jobUrlPrefix);
- }
- extractJobCounters(exportDir, jobUrlPrefix);
- extractJobTasks(exportDir, jobUrlPrefix);
- extractJobConf(exportDir, jobUrlPrefix);
- } catch (Exception e) {
- logger.warn("Failed to get mr tasks rest response.", e);
- }
- }
+ // save mr job stats
+ String jobResponse = saveHttpResponseQuietly(new File(exportDir, "job.json"), jobUrlPrefix);
+ String user = new ObjectMapper().readTree(jobResponse).path("job").path("user").textValue();
- private void extractJobCounters(File exportDir, String jobUrlPrefix) {
- String url = jobUrlPrefix + "/counters";
- String response = getHttpResponse(url);
- try {
- FileUtils.writeStringToFile(new File(exportDir, "job_counters.json"), response, Charset.defaultCharset());
- } catch (Exception e) {
- logger.warn("Failed to get mr counters rest response.", e);
- }
- }
+ String jobAttemptResponse = saveHttpResponseQuietly(new File(exportDir, "job_attempts.json"), jobUrlPrefix + "/jobattempts");
+ String nodeId = new ObjectMapper().readTree(jobAttemptResponse).path("jobAttempts").path("jobAttempt").get(0).path("nodeId").textValue();
+
+ // save mr job conf
+ saveHttpResponseQuietly(new File(exportDir, "job_conf.json"), jobUrlPrefix + "/conf");
+
+ // save mr job counters
+ saveHttpResponseQuietly(new File(exportDir, "job_counters.json"), jobUrlPrefix + "/counters");
+
+ // save task details
+ if (includeTaskDetails) {
+ extractTaskDetails(exportDir, jobUrlPrefix, jobUrlBase, nodeId, user);
+ }
- private void extractJobTasks(File exportDir, String jobUrlPrefix) {
- String url = jobUrlPrefix + "/tasks";
- String response = getHttpResponse(url);
- try {
- FileUtils.writeStringToFile(new File(exportDir, "job_tasks.json"), response, Charset.defaultCharset());
} catch (Exception e) {
- logger.warn("Failed to get mr counters rest response.", e);
+ logger.warn("Failed to get mr tasks rest response.", e);
}
}
- private void extractTaskCounters(File exportDir, String jobUrlPrefix) {
+ private void extractTaskDetails(File exportDir, String jobUrlPrefix, String jobUrlBase, String nodeId, String user) {
try {
String tasksUrl = jobUrlPrefix + "/tasks/";
- String tasksResponse = getHttpResponse(tasksUrl);
+ String tasksResponse = saveHttpResponseQuietly(new File(exportDir, "job_tasks.json"), tasksUrl);
JsonNode tasks = new ObjectMapper().readTree(tasksResponse).path("tasks").path("task");
+ // find the first start map and reduce
+ String firstStartMapId = null;
+ String firstStartReduceId = null;
+ long firstStartMapTime = Long.MAX_VALUE;
+ long firstStartReduceTime = Long.MAX_VALUE;
+
+ // find the first end map and reduce
+ String firstEndMapId = null;
+ String firstEndReduceId = null;
+ long firstEndMapTime = Long.MAX_VALUE;
+ long firstEndReduceTime = Long.MAX_VALUE;
+
+ // find the last start map and reduce
+ String lastStartMapId = null;
+ String lastStartReduceId = null;
+ long lastStartMapTime = 0L;
+ long lastStartReduceTime = 0L;
+
+ // find the last end map and reduce
+ String lastEndMapId = null;
+ String lastEndReduceId = null;
+ long lastEndMapTime = 0L;
+ long lastEndReduceTime = 0L;
+
// find the max map and reduce duation
String maxReduceId = null;
String maxMapId = null;
@@ -192,14 +238,10 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
// find the min map and reduce duration
String minReduceId = null;
String minMapId = null;
- long minMapElapsedTime = Integer.MAX_VALUE;
- long minReduceElapsedTime = Integer.MAX_VALUE;
-
- // find a normal map and reduce duration (the first one)
- String normReduceId = null;
- String normMapId = null;
- long normMapElapsedTime = 0;
- long normReduceElapsedTime = 0;
+ long minMapElapsedTime = Long.MAX_VALUE;
+ long minReduceElapsedTime = Long.MAX_VALUE;
+
+ Set<String> selectedTaskIds = Sets.newHashSet();
for (JsonNode node : tasks) {
if (node.get("type").textValue().equals("MAP")) {
if (node.get("elapsedTime").longValue() >= maxMapElapsedTime) {
@@ -212,11 +254,27 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
minMapId = node.get("id").textValue();
}
- if (normMapElapsedTime == 0) {
- normMapElapsedTime = node.get("elapsedTime").longValue();
- normMapId = node.get("id").textValue();
+ if (node.get("startTime").longValue() <= firstStartMapTime) {
+ firstStartMapTime = node.get("startTime").longValue();
+ firstStartMapId = node.get("id").textValue();
+ }
+
+ if (node.get("startTime").longValue() >= lastStartMapTime) {
+ lastStartMapTime = node.get("startTime").longValue();
+ lastStartMapId = node.get("id").textValue();
+ }
+
+ if (node.get("finishTime").longValue() <= firstEndMapTime) {
+ firstEndMapTime = node.get("finishTime").longValue();
+ firstEndMapId = node.get("id").textValue();
+ }
+
+ if (node.get("finishTime").longValue() >= lastEndMapTime) {
+ lastEndMapTime = node.get("finishTime").longValue();
+ lastEndMapId = node.get("id").textValue();
}
}
+
if (node.get("type").textValue().equals("REDUCE")) {
if (node.get("elapsedTime").longValue() >= maxReduceElapsedTime) {
maxReduceElapsedTime = node.get("elapsedTime").longValue();
@@ -228,20 +286,46 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
minReduceId = node.get("id").textValue();
}
- if (normReduceElapsedTime == 0) {
- normReduceElapsedTime = node.get("elapsedTime").longValue();
- normReduceId = node.get("id").textValue();
+ if (node.get("startTime").longValue() <= firstStartReduceTime) {
+ firstStartReduceTime = node.get("startTime").longValue();
+ firstStartReduceId = node.get("id").textValue();
+ }
+
+ if (node.get("startTime").longValue() >= lastStartReduceTime) {
+ lastStartReduceTime = node.get("startTime").longValue();
+ lastStartReduceId = node.get("id").textValue();
+ }
+
+ if (node.get("finishTime").longValue() <= firstEndReduceTime) {
+ firstEndReduceTime = node.get("finishTime").longValue();
+ firstEndReduceId = node.get("id").textValue();
+ }
+
+ if (node.get("finishTime").longValue() >= lastEndReduceTime) {
+ lastEndReduceTime = node.get("finishTime").longValue();
+ lastEndReduceId = node.get("id").textValue();
}
}
}
- File counterDir = new File(exportDir, "counters");
- FileUtils.forceMkdir(counterDir);
- extractTaskCounter(maxMapId, counterDir, tasksUrl, "max");
- extractTaskCounter(maxReduceId, counterDir, tasksUrl, "max");
- extractTaskCounter(minMapId, counterDir, tasksUrl, "min");
- extractTaskCounter(minReduceId, counterDir, tasksUrl, "min");
- extractTaskCounter(normMapId, counterDir, tasksUrl, "norm");
- extractTaskCounter(normReduceId, counterDir, tasksUrl, "norm");
+
+ selectedTaskIds.add(maxMapId);
+ selectedTaskIds.add(maxReduceId);
+ selectedTaskIds.add(minMapId);
+ selectedTaskIds.add(minReduceId);
+ selectedTaskIds.add(firstStartMapId);
+ selectedTaskIds.add(firstStartReduceId);
+ selectedTaskIds.add(lastStartMapId);
+ selectedTaskIds.add(lastStartReduceId);
+ selectedTaskIds.add(firstEndMapId);
+ selectedTaskIds.add(firstEndReduceId);
+ selectedTaskIds.add(lastEndMapId);
+ selectedTaskIds.add(lastEndReduceId);
+
+ File tasksDir = new File(exportDir, "tasks");
+ FileUtils.forceMkdir(tasksDir);
+ for (String taskId : selectedTaskIds) {
+ extractTaskDetail(taskId, nodeId, user, tasksDir, tasksUrl, jobUrlBase);
+ }
} catch (Exception e) {
logger.warn("Failed to get mr tasks rest response.", e);
}
[27/30] kylin git commit: minor change on kylin pom
Posted by li...@apache.org.
minor change on kylin pom
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c82603a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c82603a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c82603a7
Branch: refs/heads/master-hbase0.98
Commit: c82603a7f4a0f8b1128cfe057a12b922cd1282f8
Parents: 3ae6f9c
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 20 19:54:38 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 20 19:54:38 2017 +0800
----------------------------------------------------------------------
pom.xml | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c82603a7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9a56dde..dec04bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,9 +62,8 @@
<!-- Spark versions -->
<spark.version>1.6.3</spark.version>
<kryo.version>4.0.0</kryo.version>
- <!--
- <reflections.version>0.9.10</reflections.version>
- -->
+
+ <!-- <reflections.version>0.9.10</reflections.version> -->
<!-- Calcite Version -->
<calcite.version>1.11.0</calcite.version>
[12/30] kylin git commit: KYLIN-2280 Port config util
Posted by li...@apache.org.
KYLIN-2280 Port config util
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e4ed2322
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e4ed2322
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e4ed2322
Branch: refs/heads/master-hbase0.98
Commit: e4ed2322b7ff37a14a0d741766304f46ea58ba69
Parents: f852863
Author: xiefan46 <95...@qq.com>
Authored: Mon Mar 13 15:41:49 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 14:40:18 2017 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 4 +-
build/bin/kylin_port_replace_util.sh | 97 +++++++++++++++++++++++++++++++
2 files changed, 100 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e4ed2322/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index a87fa78..f30b751 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -116,7 +116,9 @@ then
echo ""
echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh stop'"
echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
- echo "Web UI is at http://<hostname>:7070/kylin"
+ kylin_server_port=`sed -n "s/<Connector port=\"\(.*\)\" protocol=\"HTTP\/1.1\"/\1/"p ${KYLIN_HOME}/tomcat/conf/server.xml`
+ kylin_server_port=`echo ${kylin_server_port}` #ignore white space
+ echo "Web UI is at http://<hostname>:${kylin_server_port}/kylin"
exit 0
# stop command
http://git-wip-us.apache.org/repos/asf/kylin/blob/e4ed2322/build/bin/kylin_port_replace_util.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin_port_replace_util.sh b/build/bin/kylin_port_replace_util.sh
new file mode 100755
index 0000000..a51e60d
--- /dev/null
+++ b/build/bin/kylin_port_replace_util.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+#exit if find error
+# ============================================================================
+
+set -o pipefail # trace ERR through pipes
+set -o errtrace # trace ERR through 'time command' and other functions
+function error() {
+ SCRIPT="$0" # script name
+ LASTLINE="$1" # line of error occurrence
+ LASTERR="$2" # error code
+ echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code ${LASTERR}"
+ exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+
+#check input parameters
+if [ $# -eq 0 ]; then
+ echo "Usage : port_offset_util.sh set PORT_OFFSET --> Modify all conflict ports base on a offset"
+ echo "Usage : port_offset_util.sh reset --> Recover to original setting"
+ exit 0
+fi
+
+#check kylin home
+if [ -z "$KYLIN_HOME" ]
+then
+ echo 'Please make sure KYLIN_HOME has been set'
+ exit 1
+else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+fi
+
+#variables
+TOMCAT_BACKUP_FILE="${KYLIN_HOME}/tomcat/conf/server.xml.backup"
+TOMCAT_CONFIG_FILE="${KYLIN_HOME}/tomcat/conf/server.xml"
+KYLIN_CONFIG_FILE="${KYLIN_HOME}/conf/kylin.properties"
+KYLIN_BACKUP_FILE="${KYLIN_HOME}/conf/kylin.properties.backup"
+TOMCAT_PORT_LIST=(9005 7070 9443 7443 9009)
+KYLIN_DEFAULT_PORT=7070
+
+if [ "$1" == "set" ]
+then
+ OFFSET=$2
+ echo "Port offset is : ${OFFSET}"
+
+ #check config file exist
+ if [ ! -f ${KYLIN_CONFIG_FILE} ] || [ ! -f ${TOMCAT_CONFIG_FILE} ]; then
+ echo "Some of the config file not exist"
+ exit 1
+ fi
+
+ #back or reset
+ if [ ! -f ${KYLIN_BACKUP_FILE} ]; then #backup if not exist
+ cp -f ${KYLIN_CONFIG_FILE} ${KYLIN_BACKUP_FILE}
+ else
+ cp -r ${KYLIN_BACKUP_FILE} ${KYLIN_CONFIG_FILE} #reset if exist
+ fi
+
+ if [ ! -f ${TOMCAT_BACKUP_FILE} ]; then #backup if not exist
+ cp -f ${TOMCAT_CONFIG_FILE} ${TOMCAT_BACKUP_FILE}
+ else
+ cp -r ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE} #reset if exist
+ fi
+
+ #replace ports in kylin.properties
+ new_kylin_port=`expr ${KYLIN_DEFAULT_PORT} + ${OFFSET}`
+
+ sed -i "s/kylin.server.cluster-servers=\(.*\).*:\(.*\)/kylin.server.cluster-servers=\1:${new_kylin_port}/g" ${KYLIN_CONFIG_FILE}
+
+ echo "New kylin port is : ${new_kylin_port}"
+
+ #replace ports in server.xml
+
+ for port in ${TOMCAT_PORT_LIST[@]}
+ do
+ new_port=`expr ${port} + ${OFFSET} `
+ #echo "Replace old port : ${port} to new port : ${new_port}"
+ sed -i "s/$port/${new_port}/g" ${TOMCAT_CONFIG_FILE}
+
+ done
+ echo "Files below modified:"
+ echo ${KYLIN_CONFIG_FILE}
+ echo ${TOMCAT_CONFIG_FILE}
+elif [ "$1" == "reset" ]
+then
+ #reset kylin.properties
+ cp -f ${KYLIN_BACKUP_FILE} ${KYLIN_CONFIG_FILE}
+ cp -f ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE}
+ rm -f ${KYLIN_BACKUP_FILE}
+ rm -f ${TOMCAT_BACKUP_FILE}
+ echo "Files below reset to original:"
+ echo ${KYLIN_CONFIG_FILE}
+ echo ${TOMCAT_CONFIG_FILE}
+else
+ echo "Unrecognized command"
+ exit 1
+fi
[25/30] kylin git commit: KYLIN-2489 upgrade zookeeper to 3.4.8
Posted by li...@apache.org.
KYLIN-2489 upgrade zookeeper to 3.4.8
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/98664f0d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/98664f0d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/98664f0d
Branch: refs/heads/master-hbase0.98
Commit: 98664f0decb0591ccc44692193145a1187201dfd
Parents: 0860048
Author: Li Yang <li...@apache.org>
Authored: Sun Mar 19 06:52:39 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Mar 19 06:52:39 2017 +0800
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/98664f0d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30506e5..9a56dde 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,7 @@
<avatica.version>1.9.0</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.8</zookeeper.version>
<curator.version>2.7.1</curator.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
[17/30] kylin git commit: KYLIN-2510 Fix unintended NPE in
CubeMetaExtractor
Posted by li...@apache.org.
KYLIN-2510 Fix unintended NPE in CubeMetaExtractor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8be842e9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8be842e9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8be842e9
Branch: refs/heads/master-hbase0.98
Commit: 8be842e9b845477541c2569ae1c2484e9d627214
Parents: dde297e
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 16 09:44:47 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 16 09:44:54 2017 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8be842e9/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index e370e48..188524d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -65,6 +65,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -172,6 +173,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
for (String projectName : projectNames.split(",")) {
ProjectInstance projectInstance = projectManager.getProject(projectName);
+ Preconditions.checkNotNull(projectInstance, "Project " + projectName + " does not exist.");
requireProject(projectInstance);
}
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
@@ -202,9 +204,6 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
private void requireProject(ProjectInstance projectInstance) throws IOException {
- if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + projectInstance.getName() + " does not exist");
- }
addRequired(projectInstance.getResourcePath());
List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
for (RealizationEntry realizationEntry : realizationEntries) {
[07/30] kylin git commit: minor, add extract all projects command
Posted by li...@apache.org.
minor, add extract all projects command
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7510c27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7510c27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7510c27
Branch: refs/heads/master-hbase0.98
Commit: e7510c270642178ca208389dbfaf856c20f9a17d
Parents: 8720362
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Mar 14 19:22:58 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 19:30:33 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 45 +++++++++++++-------
1 file changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7510c27/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 35831f7..e370e48 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Set;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
@@ -52,6 +50,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.streaming.StreamingManager;
@@ -61,8 +60,10 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -80,6 +81,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid");
@SuppressWarnings("static-access")
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_All_PROJECT = OptionBuilder.withArgName("allProjects").hasArg(false).isRequired(false).withDescription("Specify realizations in all projects to extract").create("allProjects");
@SuppressWarnings("static-access")
private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@@ -130,6 +133,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationOrProject.addOption(OPTION_CUBE);
realizationOrProject.addOption(OPTION_PROJECT);
realizationOrProject.addOption(OPTION_HYBRID);
+ realizationOrProject.addOption(OPTION_All_PROJECT);
realizationOrProject.setRequired(true);
options.addOptionGroup(realizationOrProject);
@@ -160,23 +164,15 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
- if (optionsHelper.hasOption(OPTION_PROJECT)) {
+ if (optionsHelper.hasOption(OPTION_All_PROJECT)) {
+ for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
+ requireProject(projectInstance);
+ }
+ } else if (optionsHelper.hasOption(OPTION_PROJECT)) {
String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
for (String projectName : projectNames.split(",")) {
ProjectInstance projectInstance = projectManager.getProject(projectName);
- if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + projectName + " does not exist");
- }
- addRequired(projectInstance.getResourcePath());
- List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
- for (RealizationEntry realizationEntry : realizationEntries) {
- retrieveResourcePath(getRealization(realizationEntry));
- }
- List<DataModelDesc> modelDescs = metadataManager.getModels(projectName);
- for (DataModelDesc modelDesc : modelDescs) {
- addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
- }
- addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
+ requireProject(projectInstance);
}
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
@@ -205,6 +201,22 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
engineOverwrite(new File(exportDir.getAbsolutePath()));
}
+ private void requireProject(ProjectInstance projectInstance) throws IOException {
+ if (projectInstance == null) {
+ throw new IllegalArgumentException("Project " + projectInstance.getName() + " does not exist");
+ }
+ addRequired(projectInstance.getResourcePath());
+ List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ retrieveResourcePath(getRealization(realizationEntry));
+ }
+ List<DataModelDesc> modelDescs = metadataManager.getModels(projectInstance.getName());
+ for (DataModelDesc modelDesc : modelDescs) {
+ addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ }
+ addOptional(badQueryHistoryManager.getBadQueriesForProject(projectInstance.getName()).getResourcePath());
+ }
+
private void executeExtraction(String dest) {
logger.info("The resource paths going to be extracted:");
for (String s : requiredResources) {
@@ -359,6 +371,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
+ cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
} else if (realization instanceof HybridInstance) {
[16/30] kylin git commit: KYLIN-2504 keep the engine_type property
when clone cube
Posted by li...@apache.org.
KYLIN-2504 keep the engine_type property when clone cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dde297e4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dde297e4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dde297e4
Branch: refs/heads/master-hbase0.98
Commit: dde297e48a6886f662cd153c6f834f7e26560522
Parents: 32034fc
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 15 22:25:51 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Mar 15 22:25:51 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/CubeController.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dde297e4/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index da44aec..8cdfcfb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -361,10 +360,7 @@ public class CubeController extends BasicController {
CubeDesc cubeDesc = cube.getDescriptor();
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
- KylinConfig config = cubeService.getConfig();
newCubeDesc.setName(newCubeName);
- newCubeDesc.setEngineType(config.getDefaultCubeEngine());
- newCubeDesc.setStorageType(config.getDefaultStorageEngine());
CubeInstance newCube;
try {
[19/30] kylin git commit: minor, job diag support different hadoop env
Posted by li...@apache.org.
minor, job diag support different hadoop env
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c800d6e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c800d6e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c800d6e
Branch: refs/heads/master-hbase0.98
Commit: 6c800d6e825c36ebe677b19efd7f02a7e3b9509b
Parents: 2ab9bf2
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 16 16:48:02 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 16 17:22:31 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/MrJobInfoExtractor.java | 51 ++++++++++++++------
1 file changed, 35 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c800d6e/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index b9bf2de..ca4c7e1 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -21,6 +21,7 @@ package org.apache.kylin.tool;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -46,6 +47,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class MrJobInfoExtractor extends AbstractInfoExtractor {
@@ -59,6 +61,11 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
private static final int HTTP_RETRY = 3;
+ private Map<String, String> nodeInfoMap = Maps.newHashMap();
+
+ private String jobHistoryUrlBase;
+ private String yarnMasterUrlBase;
+
public MrJobInfoExtractor() {
packageType = "MR";
@@ -71,14 +78,15 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
extractor.execute(args);
}
- private String getRestCheckUrl() {
+ private void extractRestCheckUrl() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
- Pattern pattern = Pattern.compile("(http://)(.*):.*");
+ Pattern pattern = Pattern.compile("(http://)([^:]*):([^/])*.*");
if (yarnStatusCheckUrl != null) {
Matcher m = pattern.matcher(yarnStatusCheckUrl);
if (m.matches()) {
- return m.group(1) + m.group(2) + ":19888";
+ jobHistoryUrlBase = m.group(1) + m.group(2) + ":19888";
+ yarnMasterUrlBase = m.group(1) + m.group(2) + ":" + m.group(3);
}
}
logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");
@@ -91,14 +99,15 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
}
if (StringUtils.isEmpty(rmWebHost)) {
- return null;
+ return;
}
if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
rmWebHost = "http://" + rmWebHost;
}
Matcher m = pattern.matcher(rmWebHost);
Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
- return m.group(1) + m.group(2) + ":19888";
+ yarnMasterUrlBase = rmWebHost;
+ jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address", m.group(2) + ":19888", conf);
}
private String getHttpResponse(String url) {
@@ -122,7 +131,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
return msg;
}
- private void extractTaskDetail(String taskId, String nodeId, String user, File exportDir, String taskUrl, String urlBase) throws IOException {
+ private void extractTaskDetail(String taskId, String user, File exportDir, String taskUrl, String urlBase) throws IOException {
try {
if (StringUtils.isEmpty(taskId)) {
return;
@@ -137,8 +146,9 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
String succAttemptId = taskAttempt.textValue();
String attemptInfo = saveHttpResponseQuietly(new File(destDir, "task_attempts.json"), taskUrlBase + "/attempts/" + succAttemptId);
- JsonNode attemptAttempt = new ObjectMapper().readTree(attemptInfo).path("taskAttempt").path("assignedContainerId");
- String containerId = attemptAttempt.textValue();
+ JsonNode attemptAttempt = new ObjectMapper().readTree(attemptInfo).path("taskAttempt");
+ String containerId = attemptAttempt.get("assignedContainerId").textValue();
+ String nodeId = nodeInfoMap.get(attemptAttempt.get("nodeHttpAddress").textValue());
// save task counters
saveHttpResponseQuietly(new File(destDir, "task_counters.json"), taskUrlBase + "/counters");
@@ -173,16 +183,25 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
try {
boolean includeTaskDetails = optionsHelper.hasOption(OPTION_INCLUDE_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_DETAILS)) : true;
String mrJobId = optionsHelper.getOptionValue(OPTION_MR_JOB_ID);
- String jobUrlBase = getRestCheckUrl();
- String jobUrlPrefix = jobUrlBase + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
+ extractRestCheckUrl();
+
+ Preconditions.checkNotNull(jobHistoryUrlBase);
+ Preconditions.checkNotNull(yarnMasterUrlBase);
+
+ String jobUrlPrefix = jobHistoryUrlBase + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
+
+ // cache node info
+ String nodeUrl = yarnMasterUrlBase + "/ws/v1/cluster/nodes";
+ String nodeResponse = getHttpResponse(nodeUrl);
+ JsonNode nodes = new ObjectMapper().readTree(nodeResponse).path("nodes").path("node");
+ for (JsonNode node : nodes) {
+ nodeInfoMap.put(node.path("nodeHTTPAddress").textValue(), node.path("id").textValue());
+ }
// save mr job stats
String jobResponse = saveHttpResponseQuietly(new File(exportDir, "job.json"), jobUrlPrefix);
String user = new ObjectMapper().readTree(jobResponse).path("job").path("user").textValue();
- String jobAttemptResponse = saveHttpResponseQuietly(new File(exportDir, "job_attempts.json"), jobUrlPrefix + "/jobattempts");
- String nodeId = new ObjectMapper().readTree(jobAttemptResponse).path("jobAttempts").path("jobAttempt").get(0).path("nodeId").textValue();
-
// save mr job conf
saveHttpResponseQuietly(new File(exportDir, "job_conf.json"), jobUrlPrefix + "/conf");
@@ -191,7 +210,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
// save task details
if (includeTaskDetails) {
- extractTaskDetails(exportDir, jobUrlPrefix, jobUrlBase, nodeId, user);
+ extractTaskDetails(exportDir, jobUrlPrefix, jobHistoryUrlBase, user);
}
} catch (Exception e) {
@@ -199,7 +218,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
}
}
- private void extractTaskDetails(File exportDir, String jobUrlPrefix, String jobUrlBase, String nodeId, String user) {
+ private void extractTaskDetails(File exportDir, String jobUrlPrefix, String jobUrlBase, String user) {
try {
String tasksUrl = jobUrlPrefix + "/tasks/";
String tasksResponse = saveHttpResponseQuietly(new File(exportDir, "job_tasks.json"), tasksUrl);
@@ -324,7 +343,7 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
File tasksDir = new File(exportDir, "tasks");
FileUtils.forceMkdir(tasksDir);
for (String taskId : selectedTaskIds) {
- extractTaskDetail(taskId, nodeId, user, tasksDir, tasksUrl, jobUrlBase);
+ extractTaskDetail(taskId, user, tasksDir, tasksUrl, jobUrlBase);
}
} catch (Exception e) {
logger.warn("Failed to get mr tasks rest response.", e);
[23/30] kylin git commit: minor fix unused import
Posted by li...@apache.org.
minor fix unused import
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19c87e7a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19c87e7a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19c87e7a
Branch: refs/heads/master-hbase0.98
Commit: 19c87e7a80b95a16a930feb6d0281555cd1232b5
Parents: 8b70fa5
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 17 23:33:42 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 17 23:39:22 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/metadata/model/JoinsTree.java | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/19c87e7a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 224788c..c6df52e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -30,11 +30,10 @@ import java.util.Queue;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
-public class JoinsTree implements Serializable {
+public class JoinsTree implements Serializable {
private static final long serialVersionUID = 1L;
-
+
final Map<String, Chain> tableChains = new LinkedHashMap<>();
public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
@@ -48,7 +47,7 @@ public class JoinsTree implements Serializable {
// Walk through joins to build FK table to joins mapping
HashMap<String, List<JoinDesc>> fkJoinMap = Maps.newHashMap();
int joinCount = 0;
- for (JoinDesc join: joins) {
+ for (JoinDesc join : joins) {
joinCount++;
String fkSideAlias = join.getFKSide().getAlias();
if (fkJoinMap.containsKey(fkSideAlias)) {
@@ -64,14 +63,14 @@ public class JoinsTree implements Serializable {
chainBuff.add(new Chain(rootTable, null, null));
int chainCount = 0;
while (!chainBuff.isEmpty()) {
- Chain chain= chainBuff.poll();
+ Chain chain = chainBuff.poll();
String pkSideAlias = chain.table.getAlias();
chainCount++;
tableChains.put(pkSideAlias, chain);
// this round pk side is next round's fk side
if (fkJoinMap.containsKey(pkSideAlias)) {
- for (JoinDesc join: fkJoinMap.get(pkSideAlias)) {
+ for (JoinDesc join : fkJoinMap.get(pkSideAlias)) {
chainBuff.add(new Chain(join.getPKSide(), join, chain));
}
}
@@ -149,7 +148,7 @@ public class JoinsTree implements Serializable {
public static class Chain implements Serializable {
private static final long serialVersionUID = 1L;
-
+
TableRef table; // pk side
JoinDesc join;
Chain fkSide;
[02/30] kylin git commit: minor, fix diagnosis log packaging
Posted by li...@apache.org.
minor, fix diagnosis log packaging
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08bc3386
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08bc3386
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08bc3386
Branch: refs/heads/master-hbase0.98
Commit: 08bc3386ef904f2fc2bb9482ac9ac514dfcf17f0
Parents: 9d8bf2b
Author: lidongsjtu <li...@apache.org>
Authored: Mon Mar 13 18:57:56 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Mar 13 18:58:02 2017 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/08bc3386/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
index a6d9901..6555c4d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeDescManager;
@@ -110,7 +109,8 @@ public class KylinLogExtractor extends AbstractInfoExtractor {
for (File logFile : requiredLogFiles) {
logger.info("Log file:" + logFile.getAbsolutePath());
if (logFile.exists()) {
- FileUtils.copyFileToDirectory(logFile, exportDir);
+ String cmd = String.format("cp %s %s", logFile.getAbsolutePath(), exportDir.getAbsolutePath());
+ config.getCliCommandExecutor().execute(cmd);
}
}
}
[14/30] kylin git commit: KYLIN-2508,
Trans the time to UTC time when set the range of building cube
Posted by li...@apache.org.
KYLIN-2508,Trans the time to UTC time when set the range of building cube
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3e6992da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3e6992da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3e6992da
Branch: refs/heads/master-hbase0.98
Commit: 3e6992da3b844d4642985f732c3269e354fed88c
Parents: 5155994
Author: luguosheng <55...@qq.com>
Authored: Wed Mar 15 17:21:39 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 18:35:50 2017 +0800
----------------------------------------------------------------------
webapp/app/js/directives/directives.js | 77 +++++++++++++++--------------
1 file changed, 39 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3e6992da/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index ca08493..bca3b03 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -249,49 +249,50 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
}
};
}).directive('dateTimepickerTimezone', function () {
- // this directive workaround to convert GMT0 timestamp to GMT date for datepicker
- return {
- restrict: 'A',
- priority: 1,
- require: 'ngModel',
- link: function (scope, element, attrs, ctrl) {
- ctrl.$formatters.push(function (value) {
-
- //set null for 0
- if(value===0){
- return '';
- }
-
- //return value;
- var newDate = new Date(value + (60000 * new Date().getTimezoneOffset()));
-
- var year = newDate.getFullYear();
- var month = (newDate.getMonth()+1)<10?'0'+(newDate.getMonth()+1):(newDate.getMonth()+1);
- var date = newDate.getDate()<10?'0'+newDate.getDate():newDate.getDate();
+ return {
+ restrict: 'A',
+ priority: 1,
+ require: 'ngModel',
+ link: function (scope, element, attrs, ctrl) {
+ ctrl.$formatters.push(function (value) {
- var hour = newDate.getHours()<10?'0'+newDate.getHours():newDate.getHours();
- var mins = newDate.getMinutes()<10?'0'+newDate.getMinutes():newDate.getMinutes();
- var seconds = newDate.getSeconds()<10?'0'+newDate.getSeconds():newDate.getSeconds();
+ //set null for 0
+ if(value===0){
+ return '';
+ }
- var viewVal = year+"-"+month+"-"+date+" "+hour+":"+mins+":"+seconds;
- return viewVal;
- });
+ //return value;
+ var newDate = new Date(value);
+ var year = newDate.getUTCFullYear();
+ var month = (newDate.getUTCMonth()+1)<10?'0'+(newDate.getUTCMonth()+1):(newDate.getUTCMonth()+1);
+ var date = newDate.getUTCDate()<10?'0'+newDate.getUTCDate():newDate.getUTCDate();
+ var hour = newDate.getUTCHours()<10?'0'+newDate.getUTCHours():newDate.getUTCHours();
+ var mins = newDate.getUTCMinutes()<10?'0'+newDate.getUTCMinutes():newDate.getUTCMinutes();
+ var seconds = newDate.getUTCSeconds()<10?'0'+newDate.getUTCSeconds():newDate.getUTCSeconds();
+ var viewVal = year+"-"+month+"-"+date+" "+hour+":"+mins+":"+seconds;
+ return viewVal;
+ });
- ctrl.$parsers.push(function (value) {
- var date;
- if(/^\d{4}-\d{1,2}-\d{1,2}(\s+\d{1,2}:\d{1,2}:\d{1,2})?$/.test(value)) {
- date=new Date(value);
- if(!date||date&&!date.getTime()){
- return value;
+ ctrl.$parsers.push(function (value) {
+ var date;
+ if(/^\d{4}-\d{1,2}-\d{1,2}(\s+\d{1,2}:\d{1,2}:\d{1,2})?$/.test(value)) {
+ date=new Date(value);
+ if(!date||date&&!date.getTime()){
+ return value;
+ }else{
+ var dateSplit=value.replace(/^\s+|\s+$/,'').replace(/\s+/,'-').split(/[:-]/);
+ var resultDate=[];
+ for(var i=0;i<6;i++){
+ resultDate[i]=dateSplit[i]||0;
+ }
+ return Date.UTC(resultDate[0],resultDate[1]-1,resultDate[2],resultDate[3],resultDate[4],resultDate[5]);
+ }
}else{
- return date.getTime()-(60000 * date.getTimezoneOffset());
+ return value;
}
- }else{
- return value;
- }
- });
- }
- };
+ });
+ }
+ };
}).directive("parametertree", function($compile) {
return {
restrict: "E",
[22/30] kylin git commit: KYLIN-2514 make left and inner model align
Posted by li...@apache.org.
KYLIN-2514 make left and inner model align
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b70fa52
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b70fa52
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b70fa52
Branch: refs/heads/master-hbase0.98
Commit: 8b70fa5210c7c2f95086dd2311d373277161d4ed
Parents: 14b96a8
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Mar 17 22:19:16 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 17 23:28:15 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/JoinsTree.java | 3 ++-
.../model_desc/ci_inner_join_model.json | 28 ++++++++++----------
2 files changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b70fa52/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 3c876a0..224788c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
+import java.util.ArrayDeque;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -59,7 +60,7 @@ public class JoinsTree implements Serializable {
}
// Width-first build tree (tableChains)
- Queue<Chain> chainBuff = Queues.newArrayDeque();
+ Queue<Chain> chainBuff = new ArrayDeque<Chain>();
chainBuff.add(new Chain(rootTable, null, null));
int chainCount = 0;
while (!chainBuff.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b70fa52/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
index 19cf721..b79d293 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -31,20 +31,6 @@
}
},
{
- "table": "DEFAULT.TEST_ACCOUNT",
- "alias": "SELLER_ACCOUNT",
- "kind": "FACT",
- "join": {
- "type": "INNER",
- "primary_key": [
- "SELLER_ACCOUNT.ACCOUNT_ID"
- ],
- "foreign_key": [
- "TEST_KYLIN_FACT.SELLER_ID"
- ]
- }
- },
- {
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "INNER",
@@ -119,6 +105,20 @@
"SELLER_ACCOUNT.ACCOUNT_COUNTRY"
]
}
+ },
+ {
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "SELLER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "SELLER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SELLER_ID"
+ ]
+ }
}
],
"dimensions": [
[28/30] kylin git commit: minor change on kylin pom
Posted by li...@apache.org.
minor change on kylin pom
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f72a3f6d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f72a3f6d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f72a3f6d
Branch: refs/heads/master-hbase0.98
Commit: f72a3f6d35c028ebb5c68ee6bd11b97750fbf194
Parents: c82603a
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 20 20:08:43 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 20 20:08:43 2017 +0800
----------------------------------------------------------------------
pom.xml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f72a3f6d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dec04bc..0e41dcf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -806,8 +806,7 @@
<dependencies>
<!-- the logging dependencies are inherited by all modules for their generality
- log4j and slf4j-log4j12 test scope only for UT/IT use
- -->
+ log4j and slf4j-log4j12 test scope only for UT/IT use -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
[18/30] kylin git commit: minor,
process NoSuchMethodError with lower version Kafka consumer
Posted by li...@apache.org.
minor, process NoSuchMethodError with lower version Kafka consumer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ab9bf2d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ab9bf2d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ab9bf2d
Branch: refs/heads/master-hbase0.98
Commit: 2ab9bf2d88aae2e8fa6189e8e375e140e9dbb4b5
Parents: 8be842e
Author: Billy Liu <bi...@apache.org>
Authored: Wed Mar 15 21:03:47 2017 -0700
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Mar 15 21:03:47 2017 -0700
----------------------------------------------------------------------
.../kylin/source/kafka/config/KafkaConsumerProperties.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ab9bf2d/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
index 5b3dd87..cc32ed9 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConsumerProperties.java
@@ -79,8 +79,8 @@ public class KafkaConsumerProperties {
Set<String> configNames = new HashSet<String>();
try {
configNames = ConsumerConfig.configNames();
- } catch (Exception e) {
- // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException
+ } catch (Error e) {
+ // the Kafka configNames api is supported on 0.10.1.0+, in case NoSuchMethodException which is an Error, not Exception
String[] configNamesArray = ("metric.reporters, metadata.max.age.ms, partition.assignment.strategy, reconnect.backoff.ms," + "sasl.kerberos.ticket.renew.window.factor, max.partition.fetch.bytes, bootstrap.servers, ssl.keystore.type," + " enable.auto.commit, sasl.mechanism, interceptor.classes, exclude.internal.topics, ssl.truststore.password," + " client.id, ssl.endpoint.identification.algorithm, max.poll.records, check.crcs, request.timeout.ms, heartbeat.interval.ms," + " auto.commit.interval.ms, receive.buffer.bytes, ssl.truststore.type, ssl.truststore.location, ssl.keystore.password, fetch.min.bytes," + " fetch.max.bytes, send.buffer.bytes, max.poll.interval.ms, value.deserializer, group.id, retry.backoff.ms,"
+ " ssl.secure.random.implementation, sasl.kerberos.kinit.cmd, sasl.kerberos.service.name, sasl.kerberos.ticket.renew.jitter, ssl.trustmanager.algorithm, ssl.key.password, fetch.max.wait.ms, sasl.kerberos.min.time.before.relogin, connections.max.idle.ms, session.timeout.ms, metrics.num.samples, key.deserializer, ssl.protocol, ssl.provider, ssl.enabled.protocols, ssl.keystore.location, ssl.cipher.suites, security.protocol, ssl.keymanager.algorithm, metrics.sample.window.ms, auto.offset.reset").split(",");
configNames.addAll(Arrays.asList(configNamesArray));
[04/30] kylin git commit: minor,
more log for dictionary value not exist exception
Posted by li...@apache.org.
minor, more log for dictionary value not exist exception
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b53f2b6d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b53f2b6d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b53f2b6d
Branch: refs/heads/master-hbase0.98
Commit: b53f2b6d0bed3e426fbb3ba184cb8fe44018c286
Parents: 06e4027
Author: Li Yang <li...@apache.org>
Authored: Tue Mar 14 13:40:53 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Mar 14 13:40:53 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/gridtable/CubeCodeSystem.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b53f2b6d/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
index aeeb893..aaa12a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeCodeSystem.java
@@ -19,9 +19,11 @@
package org.apache.kylin.cube.gridtable;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Map;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer;
@@ -121,7 +123,7 @@ public class CubeCodeSystem implements IGTCodeSystem {
try {
serializer.serialize(value, buf);
} catch (IllegalArgumentException ex) {
- IllegalArgumentException rewordEx = new IllegalArgumentException("Column " + col + " value '" + value + "' met dictionary error: " + ex.getMessage());
+ IllegalArgumentException rewordEx = new IllegalArgumentException("Column " + col + " value '" + toStringBinary(value) + "' met dictionary error: " + ex.getMessage());
rewordEx.setStackTrace(ex.getStackTrace());
throw rewordEx;
}
@@ -134,6 +136,14 @@ public class CubeCodeSystem implements IGTCodeSystem {
}
}
+ private String toStringBinary(Object value) {
+ if (value == null)
+ return "Null";
+ byte[] bytes;
+ bytes = value.toString().getBytes(Charset.forName("UTF-8"));
+ return Bytes.toStringBinary(bytes);
+ }
+
@Override
public Object decodeColumnValue(int col, ByteBuffer buf) {
return serializers[col].deserialize(buf);
[26/30] kylin git commit: KYLIN-2434 support config
kylin.source.hive.database-for-flat-table in Spark cubing
Posted by li...@apache.org.
KYLIN-2434 support config kylin.source.hive.database-for-flat-table in Spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ae6f9c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ae6f9c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ae6f9c9
Branch: refs/heads/master-hbase0.98
Commit: 3ae6f9c9120360cdc9fc238a2a9208fa9813aea6
Parents: 98664f0
Author: shaofengshi <sh...@apache.org>
Authored: Sat Mar 18 10:39:57 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sun Mar 19 09:47:47 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 2 +-
.../org/apache/kylin/engine/spark/SparkCubingByLayer.java | 9 +++++----
2 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ae6f9c9/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index e0b3e6c..66b154d 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -47,7 +47,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
sparkExecutable.setClassName(SparkCubingByLayer.class.getName());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid());
- sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), flatTableDesc.getTableName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_TABLE.getOpt(), seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), KylinConfig.getKylinConfPath());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ae6f9c9/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index f7ed2d0..f70fd30 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -77,22 +77,23 @@ import java.util.List;
/**
+ * Spark application to build cube with the "by-layer" algorithm. Only support source data from Hive; Metadata in HBase.
*/
public class SparkCubingByLayer extends AbstractApplication implements Serializable {
protected static final Logger logger = LoggerFactory.getLogger(SparkCubingByLayer.class);
- public static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Hive Intermediate Table").create("hiveTable");
public static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube Name").create(BatchConstants.ARG_CUBE_NAME);
public static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName("segment").hasArg().isRequired(true).withDescription("Cube Segment Id").create("segmentId");
public static final Option OPTION_CONF_PATH = OptionBuilder.withArgName("confPath").hasArg().isRequired(true).withDescription("Configuration Path").create("confPath");
public static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Cube output path").create(BatchConstants.ARG_OUTPUT);
+ public static final Option OPTION_INPUT_TABLE = OptionBuilder.withArgName("hiveTable").hasArg().isRequired(true).withDescription("Hive Intermediate Table").create("hiveTable");
private Options options;
public SparkCubingByLayer() {
options = new Options();
- options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_INPUT_TABLE);
options.addOption(OPTION_CUBE_NAME);
options.addOption(OPTION_SEGMENT_ID);
options.addOption(OPTION_CONF_PATH);
@@ -134,7 +135,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
- final String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_PATH);
+ final String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_TABLE);
final String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
final String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
final String confPath = optionsHelper.getOptionValue(OPTION_CONF_PATH);
@@ -154,7 +155,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
HiveContext sqlContext = new HiveContext(sc.sc());
- final DataFrame intermediateTable = sqlContext.table(envConfig.getHiveDatabaseForIntermediateTable() + "." + hiveTable);
+ final DataFrame intermediateTable = sqlContext.table(hiveTable);
final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
[24/30] kylin git commit: KYLIN-2514 reorder joinTable first
Posted by li...@apache.org.
KYLIN-2514 reorder joinTable first
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/08600484
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/08600484
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/08600484
Branch: refs/heads/master-hbase0.98
Commit: 0860048484639aa07f745726a58c8544a911e5f6
Parents: 19c87e7
Author: Roger Shi <ro...@hotmail.com>
Authored: Sat Mar 18 14:58:25 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 18 17:29:20 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 43 +++++++++++++++++++-
.../apache/kylin/metadata/model/JoinsTree.java | 42 ++-----------------
2 files changed, 45 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/08600484/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 0a303ec..a26ccce 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -19,11 +19,13 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
@@ -49,11 +51,11 @@ import com.google.common.collect.Sets;
public class DataModelDesc extends RootPersistentEntity {
private static final Logger logger = LoggerFactory.getLogger(DataModelDesc.class);
- public static enum TableKind implements Serializable{
+ public static enum TableKind implements Serializable {
FACT, LOOKUP
}
- public static enum RealizationCapacity implements Serializable{
+ public static enum RealizationCapacity implements Serializable {
SMALL, MEDIUM, LARGE
}
@@ -295,6 +297,7 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinTablesForUpgrade();
initTableAlias(tables);
initJoinColumns();
+ reorderJoins(tables);
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
@@ -452,6 +455,42 @@ public class DataModelDesc extends RootPersistentEntity {
joinsTree = new JoinsTree(rootFactTableRef, joins);
}
+ private void reorderJoins(Map<String, TableDesc> tables) {
+ if (joinTables.length == 0) {
+ return;
+ }
+
+ Map<String, List<JoinTableDesc>> fkMap = Maps.newHashMap();
+ for (JoinTableDesc joinTable : joinTables) {
+ JoinDesc join = joinTable.getJoin();
+ String fkSideName = join.getFKSide().getAlias();
+ if (fkMap.containsKey(fkSideName)) {
+ fkMap.get(fkSideName).add(joinTable);
+ } else {
+ List<JoinTableDesc> joinTableList = Lists.newArrayList();
+ joinTableList.add(joinTable);
+ fkMap.put(fkSideName, joinTableList);
+ }
+ }
+
+ JoinTableDesc[] orderedJoinTables = new JoinTableDesc[joinTables.length];
+ int orderedIndex = 0;
+
+ Queue<JoinTableDesc> joinTableBuff = new ArrayDeque<JoinTableDesc>();
+ TableDesc rootDesc = tables.get(rootFactTable);
+ joinTableBuff.addAll(fkMap.get(rootDesc.getName()));
+ while (!joinTableBuff.isEmpty()) {
+ JoinTableDesc head = joinTableBuff.poll();
+ orderedJoinTables[orderedIndex++] = head;
+ String headAlias = head.getJoin().getPKSide().getAlias();
+ if (fkMap.containsKey(headAlias)) {
+ joinTableBuff.addAll(fkMap.get(headAlias));
+ }
+ }
+
+ joinTables = orderedJoinTables;
+ }
+
private boolean validate() {
// ensure no dup between dimensions/metrics
http://git-wip-us.apache.org/repos/asf/kylin/blob/08600484/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index c6df52e..8e2192f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -19,17 +19,13 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.ArrayDeque;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
public class JoinsTree implements Serializable {
private static final long serialVersionUID = 1L;
@@ -44,41 +40,11 @@ public class JoinsTree implements Serializable {
Preconditions.checkState(col.isQualified());
}
- // Walk through joins to build FK table to joins mapping
- HashMap<String, List<JoinDesc>> fkJoinMap = Maps.newHashMap();
- int joinCount = 0;
+ tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null));
for (JoinDesc join : joins) {
- joinCount++;
- String fkSideAlias = join.getFKSide().getAlias();
- if (fkJoinMap.containsKey(fkSideAlias)) {
- fkJoinMap.get(fkSideAlias).add(join);
- } else {
- List<JoinDesc> joinDescList = Lists.newArrayList(join);
- fkJoinMap.put(fkSideAlias, joinDescList);
- }
- }
-
- // Width-first build tree (tableChains)
- Queue<Chain> chainBuff = new ArrayDeque<Chain>();
- chainBuff.add(new Chain(rootTable, null, null));
- int chainCount = 0;
- while (!chainBuff.isEmpty()) {
- Chain chain = chainBuff.poll();
- String pkSideAlias = chain.table.getAlias();
- chainCount++;
- tableChains.put(pkSideAlias, chain);
-
- // this round pk side is next round's fk side
- if (fkJoinMap.containsKey(pkSideAlias)) {
- for (JoinDesc join : fkJoinMap.get(pkSideAlias)) {
- chainBuff.add(new Chain(join.getPKSide(), join, chain));
- }
- }
- }
-
- // if join count not match (chain count - 1), there must be some join not take effect
- if (joinCount != (chainCount - 1)) {
- throw new IllegalArgumentException("There's some illegal Joins, please check your model");
+ TableRef pkSide = join.getPKSide();
+ Chain fkSide = tableChains.get(join.getFKSide().getAlias());
+ tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide));
}
}
[21/30] kylin git commit: KYLIN-2514 handle disordered joins in data
model
Posted by li...@apache.org.
KYLIN-2514 handle disordered joins in data model
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/14b96a86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/14b96a86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/14b96a86
Branch: refs/heads/master-hbase0.98
Commit: 14b96a86dc4cf6da78f2137838ca9827848f52be
Parents: 40249fe
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Mar 17 19:27:18 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 17 19:55:27 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/JoinsTree.java | 43 +++++++++++++++++---
.../model_desc/ci_left_join_model.json | 28 ++++++-------
2 files changed, 52 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/14b96a86/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index c7666cb..3c876a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -24,8 +24,12 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
public class JoinsTree implements Serializable {
private static final long serialVersionUID = 1L;
@@ -40,12 +44,41 @@ public class JoinsTree implements Serializable {
Preconditions.checkState(col.isQualified());
}
- tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null));
+ // Walk through joins to build FK table to joins mapping
+ HashMap<String, List<JoinDesc>> fkJoinMap = Maps.newHashMap();
+ int joinCount = 0;
+ for (JoinDesc join: joins) {
+ joinCount++;
+ String fkSideAlias = join.getFKSide().getAlias();
+ if (fkJoinMap.containsKey(fkSideAlias)) {
+ fkJoinMap.get(fkSideAlias).add(join);
+ } else {
+ List<JoinDesc> joinDescList = Lists.newArrayList(join);
+ fkJoinMap.put(fkSideAlias, joinDescList);
+ }
+ }
- for (JoinDesc join : joins) {
- TableRef pkSide = join.getPKSide();
- Chain fkSide = tableChains.get(join.getFKSide().getAlias());
- tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide));
+ // Width-first build tree (tableChains)
+ Queue<Chain> chainBuff = Queues.newArrayDeque();
+ chainBuff.add(new Chain(rootTable, null, null));
+ int chainCount = 0;
+ while (!chainBuff.isEmpty()) {
+ Chain chain= chainBuff.poll();
+ String pkSideAlias = chain.table.getAlias();
+ chainCount++;
+ tableChains.put(pkSideAlias, chain);
+
+ // this round pk side is next round's fk side
+ if (fkJoinMap.containsKey(pkSideAlias)) {
+ for (JoinDesc join: fkJoinMap.get(pkSideAlias)) {
+ chainBuff.add(new Chain(join.getPKSide(), join, chain));
+ }
+ }
+ }
+
+ // if join count not match (chain count - 1), there must be some join not take effect
+ if (joinCount != (chainCount - 1)) {
+ throw new IllegalArgumentException("There's some illegal Joins, please check your model");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/14b96a86/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
index 1b08aaf..bc5b444 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -31,20 +31,6 @@
}
},
{
- "table": "DEFAULT.TEST_ACCOUNT",
- "alias": "SELLER_ACCOUNT",
- "kind": "FACT",
- "join": {
- "type": "LEFT",
- "primary_key": [
- "SELLER_ACCOUNT.ACCOUNT_ID"
- ],
- "foreign_key": [
- "TEST_KYLIN_FACT.SELLER_ID"
- ]
- }
- },
- {
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "LEFT",
@@ -119,6 +105,20 @@
"SELLER_ACCOUNT.ACCOUNT_COUNTRY"
]
}
+ },
+ {
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "SELLER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "SELLER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SELLER_ID"
+ ]
+ }
}
],
"dimensions": [
[11/30] kylin git commit: minor,
add option to listAllCubingJobs precisely cont.
Posted by li...@apache.org.
minor, add option to listAllCubingJobs precisely cont.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8528637
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8528637
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8528637
Branch: refs/heads/master-hbase0.98
Commit: f8528637c17d0aa5260df13c0340c314a60c89c1
Parents: 242b331
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Mar 15 14:39:48 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 14:39:58 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/JobController.java | 2 +-
.../apache/kylin/rest/service/CubeService.java | 6 +-
.../apache/kylin/rest/service/JobService.java | 118 +++++++------------
.../kylin/rest/service/JobServiceTest.java | 3 +-
4 files changed, 46 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 12f9e2e..6dc7887 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -70,7 +70,7 @@ public class JobController extends BasicController {
JobTimeFilterEnum timeFilter = JobTimeFilterEnum.getByCode(jobRequest.getTimeFilter());
try {
- jobInstanceList = jobService.listAllJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
+ jobInstanceList = jobService.searchJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index ba77f06..a6310d1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -218,7 +218,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
}
@@ -248,7 +248,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
if (!cubingJobs.isEmpty()) {
throw new JobException("The cube " + cube.getName() + " has running or failed job, please discard it and try again.");
}
@@ -340,7 +340,7 @@ public class CubeService extends BasicService {
throw new InternalErrorException("Cube " + cubeName + " doesn't contain any READY segment");
}
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Enable is not allowed with a running job.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1fafb12..9836766 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -138,74 +138,6 @@ public class JobService extends BasicService implements InitializingBean {
}));
}
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) throws IOException, JobException {
- Integer limit = (null == limitValue) ? 30 : limitValue;
- Integer offset = (null == offsetValue) ? 0 : offsetValue;
- List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList, timeFilter);
- Collections.sort(jobs);
-
- if (jobs.size() <= offset) {
- return Collections.emptyList();
- }
-
- if ((jobs.size() - offset) < limit) {
- return jobs.subList(offset, jobs.size());
- }
-
- return jobs.subList(offset, offset + limit);
- }
-
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTime(new Date());
- long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
- return listCubeJobInstance(cubeName, projectName, statusList, timeStartInMillis, Long.MAX_VALUE);
- }
-
- @Deprecated
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue) throws IOException, JobException {
- Integer limit = (null == limitValue) ? 30 : limitValue;
- Integer offset = (null == offsetValue) ? 0 : offsetValue;
- List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList);
- Collections.sort(jobs);
-
- if (jobs.size() <= offset) {
- return Collections.emptyList();
- }
-
- if ((jobs.size() - offset) < limit) {
- return jobs.subList(offset, jobs.size());
- }
-
- return jobs.subList(offset, offset + limit);
- }
-
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList) {
- return listCubeJobInstance(cubeName, projectName, statusList);
- }
-
- private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList, final long timeStartInMillis, final long timeEndInMillis) {
- Set<ExecutableState> states = convertStatusEnumToStates(statusList);
- final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
- @Override
- public JobInstance apply(CubingJob cubingJob) {
- return parseToJobInstance(cubingJob, allOutputs);
- }
- }));
- }
-
- private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList) {
- Set<ExecutableState> states = convertStatusEnumToStates(statusList);
- final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs();
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
- @Override
- public JobInstance apply(CubingJob cubingJob) {
- return parseToJobInstance(cubingJob, allOutputs);
- }
- }));
- }
-
private Set<ExecutableState> convertStatusEnumToStates(List<JobStatusEnum> statusList) {
Set<ExecutableState> states;
if (statusList == null || statusList.isEmpty()) {
@@ -484,11 +416,44 @@ public class JobService extends BasicService implements InitializingBean {
return job;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs, final boolean bEqual) {
- return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs, bEqual);
+ /**
+ * currently only support substring match
+ * @return
+ */
+ public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) throws IOException, JobException {
+ Integer limit = (null == limitValue) ? 30 : limitValue;
+ Integer offset = (null == offsetValue) ? 0 : offsetValue;
+ List<JobInstance> jobs = searchJobs(cubeNameSubstring, projectName, statusList, timeFilter);
+ Collections.sort(jobs);
+
+ if (jobs.size() <= offset) {
+ return Collections.emptyList();
+ }
+
+ if ((jobs.size() - offset) < limit) {
+ return jobs.subList(offset, jobs.size());
+ }
+
+ return jobs.subList(offset, offset + limit);
+ }
+
+ private List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTime(new Date());
+ long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
+
+ long timeEndInMillis = Long.MAX_VALUE;
+ Set<ExecutableState> states = convertStatusEnumToStates(statusList);
+ final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
+ return Lists.newArrayList(FluentIterable.from(searchCubingJobs(cubeNameSubstring, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
+ @Override
+ public JobInstance apply(CubingJob cubingJob) {
+ return parseToJobInstance(cubingJob, allOutputs);
+ }
+ }));
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean bEqual) {
+ public List<CubingJob> searchCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean cubeNameExactMatch) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class)).filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
@@ -499,7 +464,7 @@ public class JobService extends BasicService implements InitializingBean {
String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
if (executableCubeName == null)
return true;
- if (bEqual)
+ if (cubeNameExactMatch)
return executableCubeName.equalsIgnoreCase(cubeName);
else
return executableCubeName.contains(cubeName);
@@ -537,15 +502,12 @@ public class JobService extends BasicService implements InitializingBean {
return results;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final boolean bEqual) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), bEqual);
- }
-
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), false);
+ return searchCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, getExecutableManager().getAllOutputs(), true);
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) {
- return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs(), false);
+ return searchCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), 0L, Long.MAX_VALUE, getExecutableManager().getAllOutputs(), true);
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 0493be6..4150808 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
+import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.junit.Assert;
@@ -44,6 +45,6 @@ public class JobServiceTest extends ServiceTestBase {
Assert.assertNotNull(jobService.getMetadataManager());
Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
Assert.assertNull(jobService.getJobInstance("job_not_exist"));
- Assert.assertNotNull(jobService.listAllJobs(null, null, null));
+ Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL));
}
}
[05/30] kylin git commit: KYLIN-2505,
remove hardcode to avoid possible syntax errors
Posted by li...@apache.org.
KYLIN-2505, remove hardcode to avoid possible syntax errors
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ce48199
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ce48199
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ce48199
Branch: refs/heads/master-hbase0.98
Commit: 3ce48199164424734696497ddd3eef90efff9563
Parents: b53f2b6
Author: Cheng Wang <ch...@kyligence.io>
Authored: Mon Mar 13 15:39:07 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 14:21:27 2017 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce48199/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 4c65bcf..f4d63d8 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -44,8 +44,9 @@ if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-#### Caution: hard code the beeline parameter by appending sample database.
- beeline ${beeline_params}"/"$sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
+ beeline_params=${beeline_params//${hive2_url}/${hive2_url}/${sample_database}}
+ beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
hive --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
[06/30] kylin git commit: minor,
add option to listAllCubingJobs precisely
Posted by li...@apache.org.
minor, add option to listAllCubingJobs precisely
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87203624
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87203624
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87203624
Branch: refs/heads/master-hbase0.98
Commit: 872036241c6f8255a07e0f160ba54c9473bf0501
Parents: 3ce4819
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Mar 14 14:52:38 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 16:43:35 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/CubeService.java | 6 ++---
.../apache/kylin/rest/service/JobService.java | 23 +++++++++++++-------
2 files changed, 18 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/87203624/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 9aa7b2a..a4f7c76 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -218,7 +218,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
}
@@ -248,7 +248,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("The cube " + cube.getName() + " has running or failed job, please discard it and try again.");
}
@@ -340,7 +340,7 @@ public class CubeService extends BasicService {
throw new InternalErrorException("Cube " + cubeName + " doesn't contain any READY segment");
}
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("Enable is not allowed with a running job.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/87203624/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 9618be1..3f17a81 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -187,7 +187,7 @@ public class JobService extends BasicService implements InitializingBean {
private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList, final long timeStartInMillis, final long timeEndInMillis) {
Set<ExecutableState> states = convertStatusEnumToStates(statusList);
final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs)).transform(new Function<CubingJob, JobInstance>() {
+ return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
return parseToJobInstance(cubingJob, allOutputs);
@@ -198,7 +198,7 @@ public class JobService extends BasicService implements InitializingBean {
private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList) {
Set<ExecutableState> states = convertStatusEnumToStates(statusList);
final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs();
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs)).transform(new Function<CubingJob, JobInstance>() {
+ return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
return parseToJobInstance(cubingJob, allOutputs);
@@ -484,11 +484,11 @@ public class JobService extends BasicService implements InitializingBean {
return job;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
- return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs);
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs, final boolean bEqual) {
+ return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs, bEqual);
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) {
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean bEqual) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class)).filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
@@ -499,7 +499,10 @@ public class JobService extends BasicService implements InitializingBean {
String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
if (executableCubeName == null)
return true;
- return executableCubeName.contains(cubeName);
+ if (bEqual)
+ return executableCubeName.equalsIgnoreCase(cubeName);
+ else
+ return executableCubeName.contains(cubeName);
} else {
return false;
}
@@ -534,11 +537,15 @@ public class JobService extends BasicService implements InitializingBean {
return results;
}
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final boolean bEqual) {
+ return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), bEqual);
+ }
+
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs());
+ return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), false);
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) {
- return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs());
+ return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs(), false);
}
}
[03/30] kylin git commit: handle cube signature inconsistency issue
with more friendly messsage
Posted by li...@apache.org.
handle cube signature inconsistency issue with more friendly messsage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/06e4027b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/06e4027b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/06e4027b
Branch: refs/heads/master-hbase0.98
Commit: 06e4027b305c2b848c9977d25912f8482a15d75d
Parents: 08bc338
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 13 21:22:01 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 13 21:35:18 2017 +0800
----------------------------------------------------------------------
build/bin/metastore.sh | 5 +++++
.../main/java/org/apache/kylin/rest/service/CubeService.java | 8 ++++----
.../main/java/org/apache/kylin/rest/service/JobService.java | 2 +-
3 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/06e4027b/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index d045ca6..cbce279 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -87,6 +87,11 @@ then
${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool reset
+elif [ "$1" == "refresh-cube-signature" ]
+then
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher
+
elif [ "$1" == "clean" ]
then
http://git-wip-us.apache.org/repos/asf/kylin/blob/06e4027b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a28cedb..9aa7b2a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -337,7 +337,7 @@ public class CubeService extends BasicService {
}
if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
- throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
+ throw new InternalErrorException("Cube " + cubeName + " doesn't contain any READY segment");
}
final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
@@ -345,7 +345,7 @@ public class CubeService extends BasicService {
throw new JobException("Enable is not allowed with a running job.");
}
if (!cube.getDescriptor().checkSignature()) {
- throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor());
+ throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures");
}
try {
@@ -444,8 +444,8 @@ public class CubeService extends BasicService {
}
}
- if(toDelete == null){
- throw new IllegalArgumentException("Cannot find segment '" + segmentName +"'");
+ if (toDelete == null) {
+ throw new IllegalArgumentException("Cannot find segment '" + segmentName + "'");
}
if (toDelete.getStatus() != SegmentStatusEnum.READY) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/06e4027b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 893d866..9618be1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -318,7 +318,7 @@ public class JobService extends BasicService implements InitializingBean {
private void checkCubeDescSignature(CubeInstance cube) {
if (!cube.getDescriptor().checkSignature())
- throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor());
+ throw new IllegalStateException("Inconsistent cube desc signature for " + cube.getDescriptor() + ", if it's right after a upgrade, please try 'Edit CubeDesc' to delete the 'signature' field. Or use 'bin/metastore.sh refresh-cube-signature' to batch refresh all cubes' signatures");
}
public JobInstance getJobInstance(String uuid) throws IOException, JobException {
[15/30] kylin git commit: KYLIN-2344 remove spark-defaults.conf
Posted by li...@apache.org.
KYLIN-2344 remove spark-defaults.conf
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/32034fc1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/32034fc1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/32034fc1
Branch: refs/heads/master-hbase0.98
Commit: 32034fc1d0cb96156da8c4b49c35f2631a59079b
Parents: 3e6992d
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 15 20:17:01 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Mar 15 20:17:19 2017 +0800
----------------------------------------------------------------------
build/deploy/spark-defaults.conf | 5 -----
build/script/download-spark.sh | 2 --
2 files changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/32034fc1/build/deploy/spark-defaults.conf
----------------------------------------------------------------------
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
deleted file mode 100644
index 78a4bc9..0000000
--- a/build/deploy/spark-defaults.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-spark.yarn.submit.file.replication=1
-spark.yarn.max.executor.failures=3
-spark.driver.extraJavaOptions=-Dhdp.version=current
-spark.yarn.am.extraJavaOptions=-Dhdp.version=current
-spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/32034fc1/build/script/download-spark.sh
----------------------------------------------------------------------
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
index 3ea7d08..b73331e 100755
--- a/build/script/download-spark.sh
+++ b/build/script/download-spark.sh
@@ -53,5 +53,3 @@ rm -rf build/spark/lib/spark-examples-*
rm -rf build/spark/examples
rm -rf build/spark/data
rm -rf build/spark/R
-
-cp build/deploy/spark-defaults.conf build/spark/conf/spark-defaults.conf
\ No newline at end of file
[08/30] kylin git commit: minor, add MR job counters in job diagnosis
Posted by li...@apache.org.
minor, add MR job counters in job diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4768c12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4768c12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4768c12
Branch: refs/heads/master-hbase0.98
Commit: d4768c12e423e96679e4d7db73c697f0378c0a94
Parents: e7510c2
Author: lidongsjtu <li...@apache.org>
Authored: Tue Mar 14 23:03:35 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Mar 14 23:05:24 2017 +0800
----------------------------------------------------------------------
build/bin/diag.sh | 2 +-
build/conf/kylin-tools-log4j.properties | 1 +
.../apache/kylin/tool/JobDiagnosisInfoCLI.java | 6 +-
.../apache/kylin/tool/MrJobInfoExtractor.java | 107 ++++++++++++++-----
4 files changed, 86 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4768c12/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index e9354a2..a995774 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -52,7 +52,7 @@ then
if [ ${#patient} -eq 36 ]; then
hbase ${KYLIN_EXTRA_START_OPTS} \
- -Dlog4j.configuration=kylin-server-log4j.properties \
+ -Dlog4j.configuration=kylin-tool-log4j.properties \
-Dcatalina.home=${tomcat_root} \
org.apache.kylin.tool.JobDiagnosisInfoCLI \
-jobId $patient \
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4768c12/build/conf/kylin-tools-log4j.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index e975d18..2ccd772 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -35,3 +35,4 @@ log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %
#log4j.logger.org.apache.hadoop=ERROR
log4j.logger.org.apache.kylin=DEBUG
log4j.logger.org.springframework=WARN
+log4j.logger.org.apache.commons.httpclient=WARN
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4768c12/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
index 638d97b..04dbef7 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobDiagnosisInfoCLI.java
@@ -192,10 +192,10 @@ public class JobDiagnosisInfoCLI extends AbstractInfoExtractor {
private void extractJobInfo(String taskId, File destDir) throws Exception {
final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo();
if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) {
- String jobId = jobInfo.get(ExecutableConstants.MR_JOB_ID);
+ String mrJobId = jobInfo.get(ExecutableConstants.MR_JOB_ID);
FileUtils.forceMkdir(destDir);
- String[] mrJobArgs = { "-destDir", destDir.getAbsolutePath(), "-compress", "false", "-submodule", "true" };
- new MrJobInfoExtractor(jobId).execute(mrJobArgs);
+ String[] mrJobArgs = { "-mrJobId", mrJobId, "-destDir", destDir.getAbsolutePath(), "-compress", "false", "-submodule", "true" };
+ new MrJobInfoExtractor().execute(mrJobArgs);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4768c12/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
index ea19885..55b54a5 100644
--- a/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/MrJobInfoExtractor.java
@@ -44,22 +44,29 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
public class MrJobInfoExtractor extends AbstractInfoExtractor {
- private String mrJobId;
- private String jobUrlPrefix;
-
private static final Logger logger = LoggerFactory.getLogger(MrJobInfoExtractor.class);
@SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_COUNTERS = OptionBuilder.withArgName("includeCounters").hasArg().isRequired(false).withDescription("Specify whether to include mr task counters to extract. Default false.").create("includeCounters");
+ private static final Option OPTION_INCLUDE_COUNTERS = OptionBuilder.withArgName("includeCounters").hasArg().isRequired(false).withDescription("Specify whether to include mr task counters to extract. Default true.").create("includeCounters");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_MR_JOB_ID = OptionBuilder.withArgName("mrJobId").hasArg().isRequired(false).withDescription("Specify MR Job Id").create("mrJobId");
- private final int HTTP_RETRY = 3;
+ private static final int HTTP_RETRY = 3;
+
+ public MrJobInfoExtractor() {
+ packageType = "MR";
+
+ options.addOption(OPTION_INCLUDE_COUNTERS);
+ options.addOption(OPTION_MR_JOB_ID);
+ }
- public MrJobInfoExtractor(String mrJobId) {
- this.mrJobId = mrJobId;
- String historyServerUrl = getRestCheckUrl();
- this.jobUrlPrefix = historyServerUrl + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
+ public static void main(String[] args) {
+ MrJobInfoExtractor extractor = new MrJobInfoExtractor();
+ extractor.execute(args);
}
private String getRestCheckUrl() {
@@ -84,13 +91,11 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
if (StringUtils.isEmpty(rmWebHost)) {
return null;
}
- if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
- //do nothing
- } else {
+ if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
rmWebHost = "http://" + rmWebHost;
}
Matcher m = pattern.matcher(rmWebHost);
- m.matches();
+ Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
return m.group(1) + m.group(2) + ":19888";
}
@@ -115,19 +120,17 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
return msg;
}
- private void extractTaskCounter(String taskId, File exportDir, String taskUrl) throws IOException {
+ private void extractTaskCounter(String taskId, File exportDir, String taskUrl, String id) throws IOException {
try {
String response = getHttpResponse(taskUrl + taskId + "/counters");
- FileUtils.writeStringToFile(new File(exportDir, taskId + ".json"), response, Charset.defaultCharset());
+ FileUtils.writeStringToFile(new File(exportDir, id + "_" + taskId + ".json"), response, Charset.defaultCharset());
} catch (Exception e) {
logger.warn("Failed to get task counters rest response" + e);
}
}
- private void extractJobConf(File exportDir) throws IOException {
+ private void extractJobConf(File exportDir, String jobUrlPrefix) throws IOException {
try {
- String jobResponse = getHttpResponse(jobUrlPrefix);
- JsonNode job = new ObjectMapper().readTree(jobResponse).path("job").get("state");
String confUrl = jobUrlPrefix + "/conf/";
String response = getHttpResponse(confUrl);
FileUtils.writeStringToFile(new File(exportDir, "job_conf.json"), response, Charset.defaultCharset());
@@ -139,47 +142,99 @@ public class MrJobInfoExtractor extends AbstractInfoExtractor {
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
try {
- boolean includeTaskCounter = optionsHelper.hasOption(OPTION_INCLUDE_COUNTERS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_COUNTERS)) : false;
+ boolean includeTaskCounter = optionsHelper.hasOption(OPTION_INCLUDE_COUNTERS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_COUNTERS)) : true;
+ String mrJobId = optionsHelper.getOptionValue(OPTION_MR_JOB_ID);
+ String jobUrlPrefix = getRestCheckUrl() + "/ws/v1/history/mapreduce/jobs/" + mrJobId;
+
if (includeTaskCounter) {
- extractTaskCounters(exportDir);
+ extractTaskCounters(exportDir, jobUrlPrefix);
}
- extractJobConf(exportDir);
+ extractJobCounters(exportDir, jobUrlPrefix);
+ extractJobConf(exportDir, jobUrlPrefix);
} catch (Exception e) {
logger.warn("Failed to get mr tasks rest response.", e);
}
}
- private void extractTaskCounters(File exportDir) {
+ private void extractJobCounters(File exportDir, String jobUrlPrefix) {
+ String url = jobUrlPrefix + "/counters";
+ String response = getHttpResponse(url);
+ try {
+ File counterDir = new File(exportDir, "counters");
+ FileUtils.forceMkdir(counterDir);
+ FileUtils.writeStringToFile(new File(exportDir, "job_counters.json"), response, Charset.defaultCharset());
+ } catch (Exception e) {
+ logger.warn("Failed to get mr counters rest response.", e);
+ }
+ }
+
+ private void extractTaskCounters(File exportDir, String jobUrlPrefix) {
try {
String tasksUrl = jobUrlPrefix + "/tasks/";
String tasksResponse = getHttpResponse(tasksUrl);
JsonNode tasks = new ObjectMapper().readTree(tasksResponse).path("tasks").path("task");
+ // find the max map and reduce duation
String maxReduceId = null;
String maxMapId = null;
long maxMapElapsedTime = 0L;
long maxReduceElapsedTime = 0L;
+ // find the min map and reduce duration
+ String minReduceId = null;
+ String minMapId = null;
+ long minMapElapsedTime = Integer.MAX_VALUE;
+ long minReduceElapsedTime = Integer.MAX_VALUE;
+
+ // find a normal map and reduce duration (the first one)
+ String normReduceId = null;
+ String normMapId = null;
+ long normMapElapsedTime = 0;
+ long normReduceElapsedTime = 0;
for (JsonNode node : tasks) {
if (node.get("type").textValue().equals("MAP")) {
if (node.get("elapsedTime").longValue() >= maxMapElapsedTime) {
maxMapElapsedTime = node.get("elapsedTime").longValue();
maxMapId = node.get("id").textValue();
}
+
+ if (node.get("elapsedTime").longValue() <= minMapElapsedTime) {
+ minMapElapsedTime = node.get("elapsedTime").longValue();
+ minMapId = node.get("id").textValue();
+ }
+
+ if (normMapElapsedTime == 0) {
+ normMapElapsedTime = node.get("elapsedTime").longValue();
+ normMapId = node.get("id").textValue();
+ }
}
if (node.get("type").textValue().equals("REDUCE")) {
if (node.get("elapsedTime").longValue() >= maxReduceElapsedTime) {
maxReduceElapsedTime = node.get("elapsedTime").longValue();
maxReduceId = node.get("id").textValue();
}
+
+ if (node.get("elapsedTime").longValue() <= minReduceElapsedTime) {
+ minReduceElapsedTime = node.get("elapsedTime").longValue();
+ minReduceId = node.get("id").textValue();
+ }
+
+ if (normReduceElapsedTime == 0) {
+ normReduceElapsedTime = node.get("elapsedTime").longValue();
+ normReduceId = node.get("id").textValue();
+ }
}
}
File counterDir = new File(exportDir, "counters");
FileUtils.forceMkdir(counterDir);
- extractTaskCounter(maxMapId, counterDir, tasksUrl);
- extractTaskCounter(maxReduceId, counterDir, tasksUrl);
+ extractTaskCounter(maxMapId, counterDir, tasksUrl, "max");
+ extractTaskCounter(maxReduceId, counterDir, tasksUrl, "max");
+ extractTaskCounter(minMapId, counterDir, tasksUrl, "min");
+ extractTaskCounter(minReduceId, counterDir, tasksUrl, "min");
+ extractTaskCounter(normMapId, counterDir, tasksUrl, "norm");
+ extractTaskCounter(normReduceId, counterDir, tasksUrl, "norm");
} catch (Exception e) {
- logger.warn("Failed to get mr tasks rest response" + e);
+ logger.warn("Failed to get mr tasks rest response.", e);
}
}
-}
+}
\ No newline at end of file