You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/12 04:05:04 UTC
[01/50] [abbrv] kylin git commit: KYLIN-2342 When
NoClassDefFoundError occurred in building cube,
no error in kylin.log [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase1.x 982e8cf2a -> c47f4ad57 (forced update)
KYLIN-2342 When NoClassDefFoundError occurred in building cube, no error in kylin.log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e043d5fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e043d5fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e043d5fa
Branch: refs/heads/master-hbase1.x
Commit: e043d5fad8ccf420b1c0e744ee87a237cd7babd7
Parents: e6a84c5
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 10:54:33 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 12:01:36 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeController.java | 4 ++--
.../org/apache/kylin/rest/controller/TableController.java | 10 ++++++----
.../org/apache/kylin/rest/init/InitialTaskManager.java | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 978f477..a9c8321 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -293,7 +293,7 @@ public class CubeController extends BasicController {
}
return jobService.submitJob(cube, startTime, endTime, startOffset, endOffset, //
sourcePartitionOffsetStart, sourcePartitionOffsetEnd, CubeBuildTypeEnum.valueOf(buildType), force, submitter);
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -656,7 +656,7 @@ public class CubeController extends BasicController {
cubeService.getCubeDescManager().updateCubeDesc(desc);
response.setProperty("result", "success");
response.setProperty("offsets", startOffsets.toString());
- } catch (Exception e) {
+ } catch (Throwable e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 1ab6ba6..ac4ee2f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -112,7 +112,7 @@ public class TableController extends BasicController {
if (request.isCalculate()) {
tableService.calculateCardinalityIfNotPresent(loaded, submitter);
}
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Failed to load Hive Table", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -133,7 +133,7 @@ public class TableController extends BasicController {
unLoadFail.add(tableName);
}
}
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Failed to unload Hive Table", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -175,7 +175,8 @@ public class TableController extends BasicController {
private List<String> showHiveDatabases() throws IOException {
try {
return tableService.getHiveDbNames();
- } catch (Exception e) {
+ } catch (Throwable e) {
+ logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
}
@@ -191,7 +192,8 @@ public class TableController extends BasicController {
private List<String> showHiveTables(@PathVariable String database) throws IOException {
try {
return tableService.getHiveTableNames(database);
- } catch (Exception e) {
+ } catch (Throwable e) {
+ logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 5cf48cb..8bba674 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -52,7 +52,7 @@ public class InitialTaskManager implements InitializingBean {
InitialTask task = (InitialTask) Class.forName(taskClass).newInstance();
logger.info("Running task: " + taskClass);
task.execute();
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Initial task failed: " + taskClass, e);
}
}
[35/50] [abbrv] kylin git commit: KYLIN-2344 Package spark into Kylin
binary package
Posted by li...@apache.org.
KYLIN-2344 Package spark into Kylin binary package
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e864cd3b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e864cd3b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e864cd3b
Branch: refs/heads/master-hbase1.x
Commit: e864cd3b1c06700b1a1054d1f520eaabedc25d82
Parents: b7d87bb
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jan 3 10:28:38 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
build/bin/check-env.sh | 6 ++
build/bin/find-spark-dependency.sh | 45 +++++++++++++++
build/bin/kylin.sh | 6 ++
build/conf/kylin-spark-conf.properties | 2 +-
build/conf/kylin.properties | 3 -
build/script/compress.sh | 5 +-
build/script/download-spark.sh | 52 +++++++++++++++++
build/script/functions.sh | 60 ++++++++++++++++++++
build/script/package.sh | 1 +
.../org/apache/kylin/common/KylinConfig.java | 14 +++++
.../apache/kylin/common/KylinConfigBase.java | 27 ++++-----
.../spark/SparkBatchCubingJobBuilder2.java | 4 +-
12 files changed, 200 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index a4003c9..e446d66 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -47,3 +47,9 @@ if [ $? != 0 ]
then
quit "Failed to create $WORKING_DIR. Please make sure the user has right to access $WORKING_DIR"
fi
+
+hadoop fs -mkdir -p $WORKING_DIR/spark-history
+if [ $? != 0 ]
+then
+ quit "Failed to create $WORKING_DIR/spark-history. Please make sure the user has right to access $WORKING_DIR"
+fi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/find-spark-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-spark-dependency.sh b/build/bin/find-spark-dependency.sh
new file mode 100644
index 0000000..6f74d8a
--- /dev/null
+++ b/build/bin/find-spark-dependency.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+spark_home=
+
+verbose Retrieving Spark dependency...
+if [ -n "$SPARK_HOME" ]
+then
+ verbose "SPARK_HOME is set to: $SPARK_HOME, use it to locate Spark dependencies."
+ spark_home=$SPARK_HOME
+fi
+
+if [ -z "$SPARK_HOME" ]
+then
+ verbose "SPARK_HOME wasn't set, use $KYLIN_HOME/spark"
+ spark_home=$KYLIN_HOME/spark
+fi
+
+spark_dependency=`find -L $spark_home -name 'spark-assembly-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$spark_dependency" ]
+then
+ quit "spark assembly lib not found"
+else
+ verbose "spark dependency: $spark_dependency"
+ export spark_dependency
+fi
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 0cdbbc6..7813b79 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -45,6 +45,11 @@ function retrieveDependency() {
source ${dir}/find-kafka-dependency.sh
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
fi
+
+ source ${dir}/find-spark-dependency.sh
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${spark_dependency}
+
+ verbose "HBASE_CLASSPATH: ${HBASE_CLASSPATH}"
}
# start command
@@ -112,6 +117,7 @@ then
-Dkylin.hive.dependency=${hive_dependency} \
-Dkylin.hbase.dependency=${hbase_dependency} \
-Dkylin.kafka.dependency=${kafka_dependency} \
+ -Dkylin.spark.dependency=${spark_dependency} \
-Dspring.profiles.active=${spring_profile} \
org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid &
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/conf/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties
index 81567bb..5e6dafe 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -20,7 +20,7 @@ spark.executor.cores=4
spark.executor.instances=8
spark.history.kerberos.keytab=none
spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
spark.driver.extraJavaOptions=-Dhdp.version=current
spark.yarn.am.extraJavaOptions=-Dhdp.version=current
spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 98b66cb..bd0bbd4 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -133,9 +133,6 @@ kylin.engine.mr.mapper-input-rows=1000000
# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
-# Spark install home, default be $KYLIN_HOME/spark/
-#kylin.engine.spark.spark-home=
-
# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
#kylin.engine.spark.properties-file=
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/script/compress.sh
----------------------------------------------------------------------
diff --git a/build/script/compress.sh b/build/script/compress.sh
index 4e3592e..39e429c 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -34,11 +34,12 @@ package_name=apache-kylin-${version}-bin
cd build/
rm -rf ${package_name}
mkdir ${package_name}
-cp -r lib tool bin conf tomcat ../examples/sample_cube commit_SHA1 ${package_name}
-rm -rf lib tomcat commit_SHA1
+cp -r lib tool bin conf tomcat spark ../examples/sample_cube commit_SHA1 ${package_name}
+rm -rf lib tomcat spark commit_SHA1
find ${package_name} -type d -exec chmod 755 {} \;
find ${package_name} -type f -exec chmod 644 {} \;
find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
+find ${package_name}/spark/bin/ -type f -exec chmod +x {} \;
mkdir -p ../dist
tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
rm -rf ${package_name}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/script/download-spark.sh
----------------------------------------------------------------------
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
new file mode 100755
index 0000000..dcbcbe7
--- /dev/null
+++ b/build/script/download-spark.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}/../..
+
+source build/script/functions.sh
+
+rm -rf build/spark
+
+spark_version="1.6.3"
+spark_pkg_md5="ce8a2e7529aac0f0175194061769dbd4"
+
+if [ ! -f "build/spark-${spark_version}-bin-hadoop2.6.tgz" ]
+then
+ echo "no binary file found"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.6.tgz || echo "Download spark failed"
+else
+ if [ `calMd5 build/spark-${spark_version}-bin-hadoop2.6.tgz | awk '{print $1}'` != "${spark_pkg_md5}" ]
+ then
+ echo "md5 check failed"
+ rm build/spark-${spark_version}-bin-hadoop2.6.tgz
+ wget --directory-prefix=build/ http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.6.tgz || echo "Download spark failed"
+
+ fi
+fi
+
+tar -zxvf build/spark-${spark_version}-bin-hadoop2.6.tgz -C build/ || { exit 1; }
+mv build/spark-${spark_version}-bin-hadoop2.6 build/spark
+
+# Remove unused components in Spark
+rm -rf build/spark/lib/spark-examples-*
+rm -rf build/spark/examples
+rm -rf build/spark/data
+rm -rf build/spark/python
+rm -rf build/spark/R
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/script/functions.sh
----------------------------------------------------------------------
diff --git a/build/script/functions.sh b/build/script/functions.sh
new file mode 100755
index 0000000..2eed617
--- /dev/null
+++ b/build/script/functions.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function checkCommandExits() {
+ echo "Checking ${1}..."
+ if [ -z "$(command -v ${1})" ]
+ then
+ echo "Please install ${1} first so that Kylin packaging can proceed"
+ exit 1
+ else
+ echo "${1} check passed"
+ fi
+}
+
+function exportProjectVersions() {
+ if [ -z "${kylin_versoin}" ]; then
+ export kylin_version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version -f kylin | grep -Ev '(^\[|Download\w+:)'`
+ echo "Apache Kylin Version: ${kylin_version}"
+ fi
+ if [ -z "${release_version}" ]; then
+ export release_version=$kap_version
+ fi
+}
+
+function detectOSType() {
+ OS_TYPE="linux"
+ if [[ `uname -a` =~ "Darwin" ]]; then
+ OS_TYPE="mac"
+ elif [[ `uname -a` =~ "Cygwin" ]]; then
+ OS_TYPE="windows"
+ fi
+ echo $OS_TYPE
+}
+
+function calMd5() {
+ OS_TYPE=`detectOSType`
+ if [[ "$OS_TYPE" == "mac" ]]; then
+ md5 -q $1
+ elif [[ "$OS_TYPE" == "windows" ]]; then
+ md5sum $1
+ else
+ md5sum $1
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/build/script/package.sh
----------------------------------------------------------------------
diff --git a/build/script/package.sh b/build/script/package.sh
index 1f9fbbd..c850ec3 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -76,6 +76,7 @@ git rev-parse HEAD >> build/commit_SHA1
sh build/script/build.sh || { exit 1; }
sh build/script/download-tomcat.sh || { exit 1; }
+sh build/script/download-spark.sh || { exit 1; }
sh build/script/prepare.sh || { exit 1; }
sh build/script/compress.sh || { exit 1; }
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index f169142..4eac92a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -187,6 +187,20 @@ public class KylinConfig extends KylinConfigBase {
return kylinConfig;
}
+ public static String getKylinConfPath() {
+ String kylinConfHome = System.getProperty(KYLIN_CONF);
+ if (!StringUtils.isEmpty(kylinConfHome)) {
+ logger.info("Use KYLIN_CONF=" + kylinConfHome);
+ return kylinConfHome;
+ }
+
+ String kylinHome = getKylinHome();
+ if (StringUtils.isEmpty(kylinHome))
+ throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them");
+
+ return kylinHome + File.separator + "conf";
+ }
+
static File getKylinPropertiesFile() {
String kylinConfHome = System.getProperty(KYLIN_CONF);
if (!StringUtils.isEmpty(kylinConfHome)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 77b1e1c..7d6ac2b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -61,6 +61,16 @@ abstract public class KylinConfigBase implements Serializable {
return kylinHome;
}
+ public static String getSparkHome() {
+ String sparkHome = System.getenv("SPARK_HOME");
+ if (StringUtils.isNotEmpty(sparkHome)) {
+ logger.info("SPARK_HOME was set to " + sparkHome);
+ return sparkHome;
+ }
+
+ return getKylinHome() + File.separator + "spark";
+ }
+
// backward compatibility check happens when properties is loaded or updated
static BackwardCompatibilityConfig BCC = new BackwardCompatibilityConfig();
@@ -729,23 +739,6 @@ abstract public class KylinConfigBase implements Serializable {
// ENGINE.SPARK
// ============================================================================
- public String getSparkHome() {
- String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark");
- File f = new File(sparkHome);
- if (f.exists()) {
- return f.getAbsolutePath();
- } else {
- String home = getKylinHome();
- f = new File(home, sparkHome);
- if (f.exists()) {
- return f.getAbsolutePath();
- }
- }
-
- throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties");
-
- }
-
public String getSparkHadoopConfDir() {
return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e864cd3b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 9532d31..c5d47e7 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -19,6 +19,7 @@
package org.apache.kylin.engine.spark;
import org.apache.hadoop.util.ClassUtil;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.EngineFactory;
@@ -52,7 +53,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), flatTableDesc.getTableName());
- sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), "/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/"); //FIXME
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), KylinConfig.getKylinConfPath());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath);
StringBuilder jars = new StringBuilder();
@@ -65,7 +66,6 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
sparkExecutable.setJars(jars.toString());
- // sparkExecutable.setJars("/Users/shishaofeng/.m2/repository/org/cloudera/htrace/htrace-core/2.01/htrace-core-2.01.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-protocol/0.98.8-hadoop2/hbase-protocol-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-common/0.98.8-hadoop2/hbase-common-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2//repository/org/apache/hbase/hbase-client/0.98.8-hadoop2/hbase-client-0.98.8-hadoop2.jar");
sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE + " with Spark");
return sparkExecutable;
[31/50] [abbrv] kylin git commit: modify default partition setting
Posted by li...@apache.org.
modify default partition setting
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cf52b49
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cf52b49
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cf52b49
Branch: refs/heads/master-hbase1.x
Commit: 2cf52b49ed9a1158cb2ec0ded6d907f90ee024fe
Parents: b9a69ef
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 19:56:10 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cf52b49/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index bd0bbd4..54430f0 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -137,13 +137,13 @@ kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
#kylin.engine.spark.properties-file=
# Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=200
+kylin.engine.spark.rdd-partition-cut-mb=50
# Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=1
+kylin.engine.spark.min-partition=10
# Max partition numbers of rdd
-kylin.engine.spark.max-partition=500
+kylin.engine.spark.max-partition=5000
### CUBE | DICTIONARY ###
[17/50] [abbrv] kylin git commit: minor, update jetty to 9.2.20
Posted by li...@apache.org.
minor, update jetty to 9.2.20
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1087f7ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1087f7ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1087f7ad
Branch: refs/heads/master-hbase1.x
Commit: 1087f7ad49f324b6ee9c1d48b4b6b0489236fe9c
Parents: 1b85f71
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 6 16:42:46 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 6 16:43:04 2017 +0800
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1087f7ad/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1d47361..22681e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,7 @@
<mrunit.version>1.1.0</mrunit.version>
<dbunit.version>2.5.2</dbunit.version>
<h2.version>1.4.192</h2.version>
- <jetty.version>9.3.10.v20160621</jetty.version>
+ <jetty.version>9.2.20.v20161216</jetty.version>
<jamm.version>0.3.1</jamm.version>
<!-- Commons -->
[30/50] [abbrv] kylin git commit: KYLIN-2371 Allow overwrite default
spark conf at cube level
Posted by li...@apache.org.
KYLIN-2371 Allow overwrite default spark conf at cube level
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee74a74e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee74a74e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee74a74e
Branch: refs/heads/master-hbase1.x
Commit: ee74a74e4d17a45b3832912381220af39e71f3d5
Parents: 66bca9a
Author: shaofengshi <sh...@apache.org>
Authored: Mon Jan 9 15:23:22 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
build/conf/kylin-spark-conf.properties | 5 +--
build/conf/kylin.properties | 6 ++--
.../apache/kylin/common/KylinConfigBase.java | 12 ++++---
.../kylin/job/constant/ExecutableConstants.java | 1 +
.../spark/SparkBatchCubingJobBuilder2.java | 2 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 7 ++--
.../kylin/engine/spark/SparkExecutable.java | 38 +++++++++++++++++---
.../test_case_data/sandbox/kylin.properties | 3 +-
8 files changed, 55 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties
index 5e6dafe..143e7e4 100644
--- a/build/conf/kylin-spark-conf.properties
+++ b/build/conf/kylin-spark-conf.properties
@@ -1,5 +1,5 @@
spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=200
+spark.yarn.executor.memoryOverhead=1024
spark.yarn.driver.memoryOverhead=384
spark.master=yarn
spark.submit.deployMode=cluster
@@ -20,7 +20,8 @@ spark.executor.cores=4
spark.executor.instances=8
spark.history.kerberos.keytab=none
spark.history.kerberos.principal=none
-#spark.yarn.jar=hdfs://namenode:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
spark.driver.extraJavaOptions=-Dhdp.version=current
spark.yarn.am.extraJavaOptions=-Dhdp.version=current
spark.executor.extraJavaOptions=-Dhdp.version=current
+#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 54430f0..6efa423 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -131,16 +131,16 @@ kylin.engine.mr.mapper-input-rows=1000000
### Spark Engine ###
# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
-kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
#kylin.engine.spark.properties-file=
# Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=50
+kylin.engine.spark.rdd-partition-cut-mb=10
# Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=10
+kylin.engine.spark.min-partition=1
# Max partition numbers of rdd
kylin.engine.spark.max-partition=5000
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7d6ac2b..36ddbf4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -695,6 +695,10 @@ abstract public class KylinConfigBase implements Serializable {
return getPropertiesByPrefix("kylin.engine.mr.config-override.");
}
+ public Map<String, String> getSparkConfigOverride() {
+ return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+ }
+
public double getDefaultHadoopJobReducerInputMB() {
return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500"));
}
@@ -739,8 +743,8 @@ abstract public class KylinConfigBase implements Serializable {
// ENGINE.SPARK
// ============================================================================
- public String getSparkHadoopConfDir() {
- return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+ public String getHadoopConfDir() {
+ return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
}
public String getSparkConfFile() {
@@ -764,7 +768,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public float getSparkRDDPartitionCutMB() {
- return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "200.0"));
+ return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0"));
}
@@ -773,7 +777,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getSparkMaxPartition() {
- return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "500"));
+ return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "5000"));
}
public boolean isSparkSanityCheckEnabled() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 11c7455..d7f6292 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -40,6 +40,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid";
public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube In-Mem";
+ public static final String STEP_NAME_BUILD_SPARK_CUBE = "Build Cube with Spark";
public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension Cuboid";
public static final String STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION = "Calculate HTable Region Splits";
public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 55e11c4..208a0c9 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -70,7 +70,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
sparkExecutable.setJars(jars.toString());
- sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE + " with Spark");
+ sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE);
return sparkExecutable;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 93cce81..c989dee 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -144,7 +144,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final String confPath = optionsHelper.getOptionValue(OPTION_CONF_PATH);
final String outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT_PATH);
- SparkConf conf = new SparkConf().setAppName("Cubing Application");
+ SparkConf conf = new SparkConf().setAppName("Cubing for:" + cubeName + ", segment " + segmentId);
//serialization conf
conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
conf.set("spark.kryo.registrationRequired", "true");
@@ -249,7 +249,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
final int totalLevels = cubeDesc.getBuildLevel();
- JavaPairRDD<ByteArray, Object[]>[] allRDDs = new JavaPairRDD[totalLevels];
+ JavaPairRDD<ByteArray, Object[]>[] allRDDs = new JavaPairRDD[totalLevels + 1];
int level = 0;
int partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
@@ -285,6 +285,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
private static void saveToHDFS(final JavaPairRDD<ByteArray, Object[]> rdd, final CubeDesc cubeDesc, final String hdfsBaseLocation, int level, Configuration conf) {
+ conf.set("dfs.replication", "2");
final String cuboidOutputPath = BatchCubingJobBuilder2.getCuboidOutputPathsByLevel(hdfsBaseLocation, level);
rdd.mapToPair(new PairFunction<Tuple2<ByteArray, Object[]>, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text>() {
BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());
@@ -403,7 +404,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
Long count = rdd.mapValues(new Function<Object[], Long>() {
@Override
public Long call(Object[] objects) throws Exception {
- return (Long) objects[countMeasureIndex]; // assume the first measure is COUNT(*)
+ return (Long) objects[countMeasureIndex];
}
}).reduce(new Function2<Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>>() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 644f73f..d892060 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -17,9 +17,10 @@
*/
package org.apache.kylin.engine.spark;
-import java.io.IOException;
+import java.io.File;
import java.util.Map;
+import jodd.util.StringUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -78,14 +79,43 @@ public class SparkExecutable extends AbstractExecutable {
String sparkConf = config.getSparkConfFile();
String jars = this.getParam(JARS);
- String jobJar = config.getKylinJobJarPath();
+ String hadoopConf = "/etc/hadoop/conf";
+ if (StringUtil.isNotEmpty(config.getHadoopConfDir())) {
+ hadoopConf = config.getHadoopConfDir();
+ } else {
+ String hiveConf = ClassLoader.getSystemClassLoader().getResource("hive-site.xml").getFile().toString();
+ File hiveConfFile = new File(hiveConf);
+ if (hiveConfFile.exists() == true) {
+ logger.info("Locate hive-site.xml in " + hiveConfFile);
+ hadoopConf = hiveConfFile.getParent();
+ }
+ }
+ logger.info("Using " + hadoopConf + " as HADOOP_CONF_DIR");
+
+ String hbaseConf = ClassLoader.getSystemClassLoader().getResource("hbase-site.xml").getFile().toString();
+ logger.info("Get hbase-site.xml location from classpath: " + hbaseConf);
+ File hbaseConfFile = new File(hbaseConf);
+ if (hbaseConfFile.exists() == false) {
+ throw new IllegalArgumentException("Couldn't find hbase-site.xml from classpath.");
+ }
+ String jobJar = config.getKylinJobJarPath();
if (StringUtils.isEmpty(jars)) {
jars = jobJar;
}
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry --properties-file %s ");
+
+ Map<String, String> sparkConfs = config.getSparkConfigOverride();
+ for (Map.Entry<String, String> entry : sparkConfs.entrySet()) {
+ stringBuilder.append(" --conf ").append(entry.getKey()).append("==").append(entry.getValue()).append(" ");
+ }
+
+ stringBuilder.append("--files %s --jars %s %s %s");
try {
- String cmd = String.format("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --properties-file %s --jars %s %s %s", config.getSparkHadoopConfDir(), config.getSparkHome(), sparkConf, jars, jobJar, formatArgs());
+ String cmd = String.format(stringBuilder.toString(),
+ hadoopConf, config.getSparkHome(), sparkConf, hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
logger.info("cmd:" + cmd);
final StringBuilder output = new StringBuilder();
CliCommandExecutor exec = new CliCommandExecutor();
@@ -98,7 +128,7 @@ public class SparkExecutable extends AbstractExecutable {
}
});
return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString());
- } catch (IOException e) {
+ } catch (Exception e) {
logger.error("error run spark job:", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee74a74e/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index a011911..d42e009 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -156,10 +156,9 @@ kylin.server.query-metrics-percentiles-intervals=60, 360, 3600
# Env DEV|QA|PROD
kylin.env=DEV
-kylin.source.hive.keep-flat-table=true
+kylin.source.hive.keep-flat-table=false
### Spark as Engine ###
-#kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox
kylin.engine.spark.spark-home=/usr/local/spark
kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties
[28/50] [abbrv] kylin git commit: KYLIN-2368 Enable Findbugs plugin
Posted by li...@apache.org.
KYLIN-2368 Enable Findbugs plugin
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75625ae5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75625ae5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75625ae5
Branch: refs/heads/master-hbase1.x
Commit: 75625ae512b3d6cc080053e4e8871f3fff816ef0
Parents: 6f8cdeb
Author: Billy Liu <bi...@apache.org>
Authored: Mon Jan 9 15:02:06 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Mon Jan 9 15:02:13 2017 +0800
----------------------------------------------------------------------
atopcalcite/pom.xml | 4 +--
core-common/pom.xml | 5 +--
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 5 +--
core-job/pom.xml | 5 +--
core-metadata/pom.xml | 5 +--
core-storage/pom.xml | 5 +--
engine-mr/pom.xml | 6 +---
engine-spark/pom.xml | 16 ++++------
jdbc/pom.xml | 3 +-
kylin-it/pom.xml | 11 ++++---
pom.xml | 73 +++++++++++++++++++++++++++++++-------------
query/pom.xml | 1 +
server-base/pom.xml | 5 +--
server/pom.xml | 2 +-
source-hive/pom.xml | 5 +--
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 5 +--
tomcat-ext/pom.xml | 8 +++--
tool/pom.xml | 11 ++++---
20 files changed, 92 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index b1bc171..b916df2 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -24,6 +24,7 @@
<artifactId>atopcalcite</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Calcite Overrides</name>
+ <description>Apache Kylin - Calcite Overrides</description>
<parent>
<groupId>org.apache.kylin</groupId>
@@ -31,9 +32,6 @@
<version>2.0.0-SNAPSHOT</version>
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.calcite</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 2265651..25b10a7 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-core-common</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Common</name>
+ <description>Apache Kylin - Core Common</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<!-- Basic Utilities -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index b7db363..0a30432 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -24,12 +24,12 @@
<artifactId>kylin-core-cube</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Cube</name>
+ <description>Apache Kylin - Core Cube</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index a74eccb..5d839dd 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-core-dictionary</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Dictionary</name>
+ <description>Apache Kylin - Core Dictionary</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index dad0428..36d34c8 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-core-job</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Job</name>
+ <description>Apache Kylin - Core Job</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index f5c8f8a..b3fe885 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-core-metadata</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Metadata</name>
+ <description>Apache Kylin - Core Metadata</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 8987608..9bd3f04 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-core-storage</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Core Storage</name>
+ <description>Apache Kylin - Core Storage</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index f31e650..b4ee16e 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -24,19 +24,15 @@
<artifactId>kylin-engine-mr</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - MapReduce Engine</name>
+ <description>Apache Kylin - MapReduce Engine</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
-
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 504a966..4921dca 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -20,22 +20,18 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-engine-spark</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Kylin - Spark Engine</name>
+ <description>Apache Kylin - Spark Engine</description>
+
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <artifactId>kylin-engine-spark</artifactId>
-
- <name>Apache Kylin - Spark Engine</name>
- <url>http://maven.apache.org</url>
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
-
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 45c212a..cd784f4 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -20,6 +20,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
<artifactId>kylin-jdbc</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - JDBC Driver</name>
@@ -29,11 +30,9 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
<properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<shadeBase>org.apache.kylin.jdbc.shaded</shadeBase>
</properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 9bc1885..080558b 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -20,15 +20,18 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-it</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Kylin - Integration Test</name>
+ <description>Apache Kylin - Integration Test</description>
+
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-it</artifactId>
- <name>Apache Kylin - Integration Test</name>
<properties>
<hdp.version/>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d883eca..841f19d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,7 +93,6 @@
<jsch.version>0.1.53</jsch.version>
<xerces.version>2.11.0</xerces.version>
<xalan.version>2.7.2</xalan.version>
- <!--<compress-lzf.version>1.0.3</compress-lzf.version>-->
<extendedset.version>1.3.4</extendedset.version>
<kryo.version>4.0.0</kryo.version>
<ehcache.version>2.10.2.2.21</ehcache.version>
@@ -622,13 +621,6 @@
<artifactId>xalan</artifactId>
<version>${xalan.version}</version>
</dependency>
- <!--
- <dependency>
- <groupId>com.ning</groupId>
- <artifactId>compress-lzf</artifactId>
- <version>${compress-lzf.version}</version>
- </dependency>
- -->
<dependency>
<groupId>com.n3twork.druid</groupId>
<artifactId>extendedset</artifactId>
@@ -901,19 +893,6 @@
<includePom>true</includePom>
</configuration>
</plugin>
- <!--<plugin>-->
- <!--<groupId>org.apache.maven.plugins</groupId>-->
- <!--<artifactId>maven-failsafe-plugin</artifactId>-->
- <!--<version>2.6</version>-->
- <!--<executions>-->
- <!--<execution>-->
- <!--<goals>-->
- <!--<goal>integration-test</goal>-->
- <!--<goal>verify</goal>-->
- <!--</goals>-->
- <!--</execution>-->
- <!--</executions>-->
- <!--</plugin>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
@@ -946,6 +925,17 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.4</version>
+ <configuration>
+ <xmlOutput>true</xmlOutput>
+ <effort>Max</effort>
+ <threshold>Low</threshold>
+ <failOnError>false</failOnError>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>net.ju-n.maven.plugins</groupId>
<artifactId>checksum-maven-plugin</artifactId>
<version>1.3</version>
@@ -998,6 +988,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ </plugin>
</plugins>
</build>
@@ -1024,6 +1018,43 @@
<module>tomcat-ext</module>
</modules>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.9</version>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>index</report>
+ <report>summary</report>
+ <report>dependency-info</report>
+ <report>project-team</report>
+ <report>scm</report>
+ <report>issue-tracking</report>
+ <report>mailing-list</report>
+ <!-- <report>dependency-management</report> -->
+ <!-- <report>dependencies</report> -->
+ <!-- <report>dependency-convergence</report> -->
+ <report>cim</report>
+ <report>plugin-management</report>
+ <report>plugins</report>
+ <report>distribution-management</report>
+ <report>license</report>
+ <report>modules</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.4</version>
+ </plugin>
+ </plugins>
+ </reporting>
+
<profiles>
<profile>
<id>sandbox</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index b7cbf6d..1dc05d1 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -20,6 +20,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+
<artifactId>kylin-query</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Query</name>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index 1147008..627ae35 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-server-base</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - REST Server Base</name>
+ <description>Apache Kylin - REST Server Base</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index ed96442..617cc85 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -24,7 +24,7 @@
<artifactId>kylin-server</artifactId>
<packaging>war</packaging>
<name>Apache Kylin - REST Server</name>
- <description>Kylin REST Service and RPC Server.</description>
+ <description>Kylin REST Service and RPC Server</description>
<parent>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index c7a413e..35027c6 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-source-hive</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Hive Source</name>
+ <description>Apache Kylin - Hive Source</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e43894d..e2fe448 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -24,12 +24,12 @@
<artifactId>kylin-source-kafka</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - Kafka Source</name>
+ <description>Apache Kylin - Kafka Source</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index b0738f6..eb2c104 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -24,17 +24,14 @@
<artifactId>kylin-storage-hbase</artifactId>
<packaging>jar</packaging>
<name>Apache Kylin - HBase Storage</name>
+ <description>Apache Kylin - HBase Storage</description>
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <properties>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/tomcat-ext/pom.xml
----------------------------------------------------------------------
diff --git a/tomcat-ext/pom.xml b/tomcat-ext/pom.xml
index 5387eb6..a00592c 100644
--- a/tomcat-ext/pom.xml
+++ b/tomcat-ext/pom.xml
@@ -21,16 +21,18 @@
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-tomcat-ext</artifactId>
<packaging>jar</packaging>
+ <name>Apache Kylin - Tomcat Extension</name>
+ <description>Apache Kylin - Tomcat Extension</description>
+
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
- <artifactId>kylin-tomcat-ext</artifactId>
- <name>Apache Kylin - Tomcat Extension</name>
-
<dependencies>
<dependency>
<groupId>org.apache.tomcat</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/75625ae5/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 8dba3f7..aae0dab 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -19,15 +19,18 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-tool</artifactId>
+ <name>Apache Kylin - Tool</name>
+ <packaging>jar</packaging>
+ <description>Apache Kylin - Tool</description>
+
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>kylin-tool</artifactId>
- <name>Apache Kylin - Tool</name>
<dependencies>
<dependency>
[18/50] [abbrv] kylin git commit: KYLIN-2337 Remove expensive
toString in SortedIteratorMergerWithLimit
Posted by li...@apache.org.
KYLIN-2337 Remove expensive toString in SortedIteratorMergerWithLimit
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7509beee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7509beee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7509beee
Branch: refs/heads/master-hbase1.x
Commit: 7509beeeaaf1d4a50bb82fd83dff5516276b9b9e
Parents: 1087f7a
Author: kangkaisen <ka...@live.com>
Authored: Thu Dec 29 21:37:26 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 09:35:00 2017 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java | 1 +
.../kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7509beee/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 4d26029..f4480c8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -234,6 +234,7 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
return toString(info.colAll);
}
+ /** toString for MemoryHungry Measure is expensive, please invoke carefully */
public String toString(ImmutableBitSet selectedColumns) {
Object[] values = new Object[selectedColumns.cardinality()];
getValues(selectedColumns, values);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7509beee/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
index 2f2691e..6a7e713 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
@@ -126,7 +126,7 @@ public class SortedIteratorMergerWithLimit<E extends Cloneable> extends SortedIt
//TODO: remove this check when validated
if (last != null) {
- Preconditions.checkState(comparator.compare(last, fetched) <= 0, "Not sorted! last: " + last + " fetched: " + fetched);
+ Preconditions.checkState(comparator.compare(last, fetched) <= 0, "Not sorted!");
}
last = fetched;
[12/50] [abbrv] kylin git commit: KYLIN-2361 Upgrade to Tomcat 8.5.9
Posted by li...@apache.org.
KYLIN-2361 Upgrade to Tomcat 8.5.9
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f267ba9d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f267ba9d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f267ba9d
Branch: refs/heads/master-hbase1.x
Commit: f267ba9dcb3db317d07eb367ddff668e03291519
Parents: f573486
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 6 10:42:48 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 6 10:42:48 2017 +0800
----------------------------------------------------------------------
build/deploy/context.xml | 47 ++++++-------
build/deploy/server.xml | 72 +++++++++++++-------
build/script/download-tomcat.sh | 14 ++--
pom.xml | 2 +-
.../java/org/apache/kylin/rest/DebugTomcat.java | 10 +--
.../kylin/ext/CustomizedWebappClassloader.java | 5 +-
6 files changed, 82 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/context.xml
----------------------------------------------------------------------
diff --git a/build/deploy/context.xml b/build/deploy/context.xml
index 5d1bedf..e8b4cf1 100644
--- a/build/deploy/context.xml
+++ b/build/deploy/context.xml
@@ -1,38 +1,31 @@
-<?xml version='1.0' encoding='utf-8'?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<!-- The contents of this file will be loaded for each web application -->
<Context>
- <!-- Default set of monitored resources -->
+ <!-- Default set of monitored resources. If one of these changes, the -->
+ <!-- web application will be reloaded. -->
<WatchedResource>WEB-INF/web.xml</WatchedResource>
+ <WatchedResource>${catalina.base}/conf/web.xml</WatchedResource>
<!-- Uncomment this to disable session persistence across Tomcat restarts -->
<!--
<Manager pathname="" />
-->
-
- <!-- Uncomment this to enable Comet connection tacking (provides events
- on session expiration as well as webapp lifecycle) -->
- <!--
- <Valve className="org.apache.catalina.valves.CometConnectionManagerValve" />
- -->
-
<Loader loaderClass="org.apache.kylin.ext.CustomizedWebappClassloader"/>
-
-</Context>
\ No newline at end of file
+</Context>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/deploy/server.xml
----------------------------------------------------------------------
diff --git a/build/deploy/server.xml b/build/deploy/server.xml
index 72a87d0..bea6146 100644
--- a/build/deploy/server.xml
+++ b/build/deploy/server.xml
@@ -1,4 +1,4 @@
-<?xml version='1.0' encoding='utf-8'?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -20,17 +20,16 @@
Documentation at /docs/config/server.html
-->
<Server port="9005" shutdown="SHUTDOWN">
+ <Listener className="org.apache.catalina.startup.VersionLoggerListener" />
<!-- Security listener. Documentation at /docs/config/listeners.html
<Listener className="org.apache.catalina.security.SecurityListener" />
-->
<!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on"/>
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener"/>
+ <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
<!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
- <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
- <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener"/>
+ <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
+ <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
+ <Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
<!-- Global JNDI resources
Documentation at /docs/jndi-resources-howto.html
@@ -43,7 +42,7 @@
type="org.apache.catalina.UserDatabase"
description="User database that can be updated and saved"
factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml"/>
+ pathname="conf/tomcat-users.xml" />
</GlobalNamingResources>
<!-- A "Service" is a collection of one or more "Connectors" that share
@@ -62,14 +61,14 @@
<!-- A "Connector" represents an endpoint by which requests are received
and responses are returned. Documentation at :
- Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
+ Java HTTP Connector: /docs/config/http.html
Java AJP Connector: /docs/config/ajp.html
APR (HTTP/AJP) Connector: /docs/apr.html
- Define a non-SSL HTTP/1.1 Connector on port 8080
+ Define a non-SSL/TLS HTTP/1.1 Connector on port 8080
-->
<Connector port="7070" protocol="HTTP/1.1"
connectionTimeout="20000"
- redirectPort="7443"
+ redirectPort="9443"
compression="on"
compressionMinSize="2048"
noCompressionUserAgents="gozilla,traviata"
@@ -82,21 +81,44 @@
connectionTimeout="20000"
redirectPort="8443" />
-->
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the BIO implementation that requires the JSSE
- style configuration. When using the APR/native implementation, the
- OpenSSL style configuration is required as described in the APR/native
- documentation -->
+ <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443
+ This connector uses the NIO implementation. The default
+ SSLImplementation will depend on the presence of the APR/native
+ library and the useOpenSSL attribute of the
+ AprLifecycleListener.
+ Either JSSE or OpenSSL style configuration may be used regardless of
+ the SSLImplementation selected. JSSE style configuration is used below.
+ -->
<!--
- <Connector port="7443" protocol="org.apache.coyote.http11.Http11Protocol"
- maxThreads="150" SSLEnabled="true" scheme="https" secure="true"
- keystoreFile="${user.home}/.keystore" keystorePass=""
- clientAuth="false" sslProtocol="TLS" />
-
+ <Connector port="8443" protocol="org.apache.coyote.http11.Http11NioProtocol"
+ maxThreads="150" SSLEnabled="true">
+ <SSLHostConfig>
+ <Certificate certificateKeystoreFile="conf/localhost-rsa.jks"
+ type="RSA" />
+ </SSLHostConfig>
+ </Connector>
+ -->
+ <!-- Define a SSL/TLS HTTP/1.1 Connector on port 8443 with HTTP/2
+ This connector uses the APR/native implementation which always uses
+ OpenSSL for TLS.
+ Either JSSE or OpenSSL style configuration may be used. OpenSSL style
+ configuration is used below.
+ -->
+ <!--
+ <Connector port="8443" protocol="org.apache.coyote.http11.Http11AprProtocol"
+ maxThreads="150" SSLEnabled="true" >
+ <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
+ <SSLHostConfig>
+ <Certificate certificateKeyFile="conf/localhost-rsa-key.pem"
+ certificateFile="conf/localhost-rsa-cert.pem"
+ certificateChainFile="conf/localhost-rsa-chain.pem"
+ type="RSA" />
+ </SSLHostConfig>
+ </Connector>
-->
<!-- Define an AJP 1.3 Connector on port 8009 -->
- <Connector port="9009" protocol="AJP/1.3" redirectPort="9443"/>
+ <Connector port="9009" protocol="AJP/1.3" redirectPort="9443" />
<!-- An Engine represents the entry point (within Catalina) that processes
@@ -128,7 +150,7 @@
resourceName="UserDatabase"/>
</Realm>
- <Host name="localhost" appBase="webapps"
+ <Host name="localhost" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<!-- SingleSignOn valve, share authentication between web applications
@@ -141,8 +163,8 @@
Documentation at: /docs/config/valve.html
Note: The pattern used is equivalent to using pattern="common" -->
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt"
- pattern="%h %l %u %t "%r" %s %b"/>
+ prefix="localhost_access_log" suffix=".txt"
+ pattern="%h %l %u %t "%r" %s %b" />
</Host>
</Engine>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index 9dd26f0..c97c3ce 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -27,22 +27,22 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-if [ ! -f "build/apache-tomcat-7.0.69.tar.gz" ]
+if [ ! -f "build/apache-tomcat-8.5.9.tar.gz" ]
then
echo "no binary file found"
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz || echo "download tomcat failed"
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.gz || echo "download tomcat failed"
else
- if [ `md5cmd build/apache-tomcat-7.0.69.tar.gz | awk '{print $1}'` != "10a071e5169a1a8b14ff35a0ad181052" ]
+ if [ `md5cmd build/apache-tomcat-8.5.9.tar.gz | awk '{print $1}'` != "b41270a64b7774c964e4bec813eea2ed" ]
then
echo "md5 check failed"
- rm build/apache-tomcat-7.0.69.tar.gz
- wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-7/v7.0.69/bin/apache-tomcat-7.0.69.tar.gz || echo "download tomcat failed"
+ rm build/apache-tomcat-8.5.9.tar.gz
+ wget --directory-prefix=build/ http://archive.apache.org/dist/tomcat/tomcat-8/v8.5.9/bin/apache-tomcat-8.5.9.tar.g || echo "download tomcat failed"
fi
fi
unalias md5cmd
-tar -zxvf build/apache-tomcat-7.0.69.tar.gz -C build/
-mv build/apache-tomcat-7.0.69 build/tomcat
+tar -zxvf build/apache-tomcat-8.5.9.tar.gz -C build/
+mv build/apache-tomcat-8.5.9 build/tomcat
rm -rf build/tomcat/webapps/*
mv build/tomcat/conf/server.xml build/tomcat/conf/server.xml.bak
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b323cf..1d47361 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,7 +102,7 @@
<cglib.version>3.2.4</cglib.version>
<supercsv.version>2.4.0</supercsv.version>
<cors.version>2.5</cors.version>
- <tomcat.version>7.0.69</tomcat.version>
+ <tomcat.version>8.5.9</tomcat.version>
<!-- REST Service -->
<spring.framework.version>3.2.17.RELEASE</spring.framework.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 48cb91b..4f49402 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -18,18 +18,18 @@
package org.apache.kylin.rest;
-import java.io.File;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
-import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.util.descriptor.web.ErrorPage;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
public class DebugTomcat {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f267ba9d/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
----------------------------------------------------------------------
diff --git a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
index 5116849..23bde2c 100644
--- a/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
+++ b/tomcat-ext/src/main/java/org/apache/kylin/ext/CustomizedWebappClassloader.java
@@ -45,8 +45,7 @@ public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
* @param name class name
* @return true if the class should be filtered
*/
- protected boolean filter(String name) {
-
+ protected boolean filter(String name, boolean isClassName) {
if (name == null)
return false;
@@ -63,7 +62,7 @@ public class CustomizedWebappClassloader extends ParallelWebappClassLoader {
return true;
}
- return false;
+ return super.filter(name, isClassName);
}
}
[14/50] [abbrv] kylin git commit: KYLIN-2361,
fix redundency dependency introduced by Tomcat 8
Posted by li...@apache.org.
KYLIN-2361, fix redundency dependency introduced by Tomcat 8
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/40ede31b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/40ede31b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/40ede31b
Branch: refs/heads/master-hbase1.x
Commit: 40ede31bf538efb60c879914a6387f1eb107c5f1
Parents: 49de556
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 6 15:03:50 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 6 15:04:10 2017 +0800
----------------------------------------------------------------------
server/pom.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/40ede31b/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 981d64c..ed96442 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -229,6 +229,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jetty-all</artifactId>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ </exclusion>
</exclusions>
</dependency>
[06/50] [abbrv] kylin git commit: KYLIN-2333 fix
ArrayIndexOutOfBoundsException in JobBuilderSupport
Posted by li...@apache.org.
KYLIN-2333 fix ArrayIndexOutOfBoundsException in JobBuilderSupport
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5683954b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5683954b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5683954b
Branch: refs/heads/master-hbase1.x
Commit: 5683954bafec3fb02cc2ccda56387dacceb6b593
Parents: b736175
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 16:32:28 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 16:32:28 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5683954b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 122b8ba..86ac880 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -193,7 +193,7 @@ public class JobBuilderSupport {
}
public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
- String[] paths = new String[levels];
+ String[] paths = new String[levels + 1];
for (int i = 0; i <= levels; i++) {
if (i == 0) {
paths[i] = cuboidRootPath + "base_cuboid";
[43/50] [abbrv] kylin git commit: rollback ITEM_COUNT to INT type
Posted by li...@apache.org.
rollback ITEM_COUNT to INT type
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78d7086e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78d7086e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78d7086e
Branch: refs/heads/master-hbase1.x
Commit: 78d7086e51c9311ab25b17f05206766d8a279afa
Parents: 521dfff
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 11 09:40:55 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 11 09:40:55 2017 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/78d7086e/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index df53de9..2b33e66 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
}, {
"id" : "10",
"name" : "ITEM_COUNT",
- "datatype" : "bigint",
+ "datatype" : "int",
"data_gen" : "RAND"
}, {
"id" : "11",
[38/50] [abbrv] kylin git commit: KYLIN-2348 allow multiple models
serving one query
Posted by li...@apache.org.
KYLIN-2348 allow multiple models serving one query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3a36c66a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3a36c66a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3a36c66a
Branch: refs/heads/master-hbase1.x
Commit: 3a36c66a1c2bbb372b0a3f3243f181332d7ed882
Parents: 00c8f31
Author: Li Yang <li...@apache.org>
Authored: Tue Jan 10 11:32:58 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 10 11:44:17 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/Dictionary.java | 2 -
.../apache/kylin/query/ITKylinQueryTest.java | 16 +++++--
.../org/apache/kylin/query/KylinTestBase.java | 1 +
.../src/test/resources/query/sql/query101.sql | 18 +++++++
.../src/test/resources/query/sql/query102.sql | 18 +++++++
.../src/test/resources/query/sql/query103.sql | 18 +++++++
.../resources/query/sql_multi_model/query01.sql | 49 ++++++++++++++++++++
.../relnode/OLAPToEnumerableConverter.java | 18 +++----
.../kylin/query/routing/ModelChooser.java | 43 +++++++++++++----
.../apache/kylin/query/schema/OLAPTable.java | 8 ++--
10 files changed, 161 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 03996a7..1e172bc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -254,6 +254,4 @@ abstract public class Dictionary<T> implements Serializable {
*/
public abstract void readFields(DataInput in) throws IOException;
-
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 840a03f..aaf0925 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -47,7 +47,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
@@ -149,7 +149,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/temp/sample.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model/query01.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
@@ -270,7 +270,17 @@ public class ITKylinQueryTest extends KylinTestBase {
this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_intersect_count");
}
}
-
+
+
+ @Test
+ public void testMultiModelQuery() throws Exception {
+ if ("left".equalsIgnoreCase(joinType)) {
+ joinType = "default";
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_multi_model", null, true);
+ joinType = "left";
+ }
+ }
+
@Test
public void testDimDistinctCountQuery() throws Exception {
execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_distinct_dim", null, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 4f1f591..ae69f09 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -335,6 +335,7 @@ public class KylinTestBase {
printInfo("Query Result from H2 - " + queryName);
H2Connection h2Conn = new H2Connection(h2Connection, null);
h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
executeQuery(h2Conn, queryName, sql, needSort);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query101.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query101.sql b/kylin-it/src/test/resources/query/sql/query101.sql
index fb42bca..765d26d 100644
--- a/kylin-it/src/test/resources/query/sql/query101.sql
+++ b/kylin-it/src/test/resources/query/sql/query101.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query102.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query102.sql b/kylin-it/src/test/resources/query/sql/query102.sql
index bd1e15e..6073c98 100644
--- a/kylin-it/src/test/resources/query/sql/query102.sql
+++ b/kylin-it/src/test/resources/query/sql/query102.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql/query103.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query103.sql b/kylin-it/src/test/resources/query/sql/query103.sql
index c5f9bf9..6c8ce9e 100644
--- a/kylin-it/src/test/resources/query/sql/query103.sql
+++ b/kylin-it/src/test/resources/query/sql/query103.sql
@@ -1,3 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
select meta_categ_name, count(1) as cnt, sum(price) as GMV
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_multi_model/query01.sql b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
new file mode 100644
index 0000000..cb0e2f2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_multi_model/query01.sql
@@ -0,0 +1,49 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select
+ l.cal_dt
+ , sum(left_join_gvm) as left_join_sum
+ , sum(inner_join_gvm) as inner_join_sum
+from
+ (
+ select test_kylin_fact.cal_dt, sum(price) as left_join_gvm
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+
+ group by test_kylin_fact.cal_dt
+ ) l
+ inner join
+ (
+ select test_kylin_fact.cal_dt, sum(price) as inner_join_gvm
+
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+
+ group by test_kylin_fact.cal_dt
+ ) i
+ on l.cal_dt = i.cal_dt
+group by
+ l.cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 8f01a74..0468c91 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
@@ -79,19 +80,11 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
// identify model
List<OLAPContext> contexts = listContextsHavingScan();
- Set<IRealization> candidates = ModelChooser.selectModel(contexts);
+ IdentityHashMap<OLAPContext, Set<IRealization>> candidates = ModelChooser.selectModel(contexts);
// identify realization for each context
- for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
-
- // Context has no table scan is created by OLAPJoinRel which looks like
- // (sub-query) as A join (sub-query) as B
- // No realization needed for such context.
- if (context.firstTableScan == null) {
- continue;
- }
-
- IRealization realization = QueryRouter.selectRealization(context, candidates);
+ for (OLAPContext context : contexts) {
+ IRealization realization = QueryRouter.selectRealization(context, candidates.get(context));
context.realization = realization;
doAccessControl(context);
}
@@ -115,6 +108,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
}
private List<OLAPContext> listContextsHavingScan() {
+ // Context has no table scan is created by OLAPJoinRel which looks like
+ // (sub-query) as A join (sub-query) as B
+ // No realization needed for such context.
int size = OLAPContext.getThreadLocalContexts().size();
List<OLAPContext> result = Lists.newArrayListWithCapacity(size);
for (int i = 0; i < size; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index ccb2900..2517dc5 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -19,6 +19,7 @@
package org.apache.kylin.query.routing;
import java.util.Comparator;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -39,14 +40,39 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPTableScan;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class ModelChooser {
- // select a model that satisfies all the contexts
- public static Set<IRealization> selectModel(List<OLAPContext> contexts) {
+ // select models for given contexts, return realization candidates for each context
+ public static IdentityHashMap<OLAPContext, Set<IRealization>> selectModel(List<OLAPContext> contexts) {
+
+ IdentityHashMap<OLAPContext, Set<IRealization>> candidates = new IdentityHashMap<>();
+
+ // attempt one model for all contexts
+ Set<IRealization> reals = attemptSelectModel(contexts);
+ if (reals != null) {
+ for (OLAPContext ctx : contexts) {
+ candidates.put(ctx, reals);
+ }
+ return candidates;
+ }
+
+ // try different model for different context
+ for (OLAPContext ctx : contexts) {
+ reals = attemptSelectModel(ImmutableList.of(ctx));
+ if (reals == null)
+ throw new NoRealizationFoundException("No model found for" + toErrorMsg(ctx));
+
+ candidates.put(ctx, reals);
+ }
+ return candidates;
+ }
+
+ private static Set<IRealization> attemptSelectModel(List<OLAPContext> contexts) {
Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts);
for (DataModelDesc model : modelMap.keySet()) {
@@ -57,17 +83,14 @@ public class ModelChooser {
return modelMap.get(model);
}
}
-
- throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts));
+ return null;
}
- private static String toErrorMsg(List<OLAPContext> contexts) {
+ private static String toErrorMsg(OLAPContext ctx) {
StringBuilder buf = new StringBuilder();
- for (OLAPContext ctx : contexts) {
- buf.append(", ").append(ctx.firstTableScan);
- for (JoinDesc join : ctx.joins)
- buf.append(", ").append(join);
- }
+ buf.append(ctx.firstTableScan);
+ for (JoinDesc join : ctx.joins)
+ buf.append(", ").append(join);
return buf.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3a36c66a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 85996f4..fe6a806 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -186,14 +186,14 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
//to avoid overflow we upgrade x's type to long
//this includes checking two parts:
//1. sum measures in cubes:
- HashSet<ColumnDesc> updateColumns = Sets.newHashSet();
+ HashSet<ColumnDesc> upgradeCols = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
FunctionDesc func = m.getFunction();
if (func.getReturnDataType() != func.getRewriteFieldType() && //
func.getReturnDataType().isBigInt() && //
func.getRewriteFieldType().isIntegerFamily()) {
- updateColumns.add(func.getParameter().getColRefs().get(0).getColumnDesc());
+ upgradeCols.add(func.getParameter().getColRefs().get(0).getColumnDesc());
}
}
}
@@ -204,12 +204,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
for (String metricColumn : model.getMetrics()) {
TblColRef col = model.findColumn(metricColumn);
if (col.getTable().equals(sourceTable.getIdentity()) && col.getType().isIntegerFamily() && !col.getType().isBigInt())
- updateColumns.add(col.getColumnDesc());
+ upgradeCols.add(col.getColumnDesc());
}
}
}
- for (ColumnDesc upgrade : updateColumns) {
+ for (ColumnDesc upgrade : upgradeCols) {
int index = tableColumns.indexOf(upgrade);
if (index < 0) {
throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");
[03/50] [abbrv] kylin git commit: KYLIN-2358 CuboidReducer has too
many if (aggrMask[i]) checks
Posted by li...@apache.org.
KYLIN-2358 CuboidReducer has too many if (aggrMask[i]) checks
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b5257a1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b5257a1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b5257a1
Branch: refs/heads/master-hbase1.x
Commit: 4b5257a1dc4b1cf90804e5273f27944de30b8fbf
Parents: 9c98574
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 13:49:08 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 13:49:08 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/MeasureAggregators.java | 9 +++----
.../kylin/engine/mr/steps/CuboidReducer.java | 26 ++++++++++++--------
2 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 2b28302..8951b69 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -77,13 +77,10 @@ public class MeasureAggregators implements Serializable {
}
}
- public void aggregate(Object[] values, boolean[] aggrMask) {
+ public void aggregate(Object[] values, int[] measures) {
assert values.length == descLength;
- assert aggrMask.length == descLength;
-
- for (int i = 0; i < descLength; i++) {
- if (aggrMask[i])
- aggs[i].aggregate(values[i]);
+ for (int i = 0; i < measures.length; i++) {
+ aggs[measures[i]].aggregate(values[measures[i]]);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b5257a1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index afd29e3..495be77 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
+import com.google.common.collect.Lists;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeManager;
@@ -51,7 +52,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
private MeasureAggregators aggs;
private int cuboidLevel;
- private boolean[] needAggr;
+ private int[] needAggrMeasures;
private Object[] input;
private Object[] result;
private int vcounter;
@@ -76,13 +77,22 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
input = new Object[measuresDescs.size()];
result = new Object[measuresDescs.size()];
- needAggr = new boolean[measuresDescs.size()];
- if (cuboidLevel > 0) {
- for (int i = 0; i < measuresDescs.size(); i++) {
- needAggr[i] = !measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
+ List<Integer> needAggMeasuresList = Lists.newArrayList();
+ for (int i = 0; i < measuresDescs.size(); i++) {
+ if (cuboidLevel == 0) {
+ needAggMeasuresList.add(i);
+ } else {
+ if (!measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid()) {
+ needAggMeasuresList.add(i);
+ }
}
}
+
+ needAggrMeasures = new int[needAggMeasuresList.size()];
+ for (int i = 0; i < needAggMeasuresList.size(); i++) {
+ needAggrMeasures[i] = needAggMeasuresList.get(i);
+ }
}
@Override
@@ -94,11 +104,7 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
logger.info("Handling value with ordinal (This is not KV number!): " + vcounter);
}
codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input);
- if (cuboidLevel > 0) {
- aggs.aggregate(input, needAggr);
- } else {
- aggs.aggregate(input);
- }
+ aggs.aggregate(input, needAggrMeasures);
}
aggs.collectStates(result);
[39/50] [abbrv] kylin git commit: minor,
move duplicated ByteBufferBackedInputStream to common package
Posted by li...@apache.org.
minor, move duplicated ByteBufferBackedInputStream to common package
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0aadb12
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0aadb12
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0aadb12
Branch: refs/heads/master-hbase1.x
Commit: b0aadb12d1e25005445eda0d912c63a941ad929a
Parents: 3a36c66
Author: gaodayue <ga...@meituan.com>
Authored: Tue Jan 10 11:51:38 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Tue Jan 10 11:52:04 2017 +0800
----------------------------------------------------------------------
.../util/ByteBufferBackedInputStream.java | 61 ++++++++++++++++++++
.../kylin/measure/bitmap/BitmapCounter.java | 38 +-----------
.../measure/bitmap/BitmapAggregatorTest.java | 2 +-
.../kylin/measure/bitmap/BitmapCounterTest.java | 2 +-
.../measure/bitmap/BitmapSerializerTest.java | 2 +-
.../source/kafka/TimedJsonStreamParser.java | 2 +-
.../kafka/util/ByteBufferBackedInputStream.java | 52 -----------------
7 files changed, 66 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
new file mode 100644
index 0000000..7332ff2
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteBufferBackedInputStream.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * Utility to view content available in a {@link ByteBuffer} as a {@link InputStream}.
+ *
+ * <b>Not thread-safe</b>
+ */
+public class ByteBufferBackedInputStream extends InputStream {
+ private final ByteBuffer buffer;
+
+ public ByteBufferBackedInputStream(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return buffer.remaining();
+ }
+
+ @Override
+ public int read() throws IOException {
+ return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (len == 0) {
+ return 0;
+ }
+
+ if (!buffer.hasRemaining()) {
+ return -1;
+ }
+
+ len = Math.min(buffer.remaining(), len);
+ buffer.get(b, off, len);
+ return len;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 43ee506..19d7f5d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -22,10 +22,10 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
/**
@@ -84,18 +84,6 @@ public class BitmapCounter implements Comparable<BitmapCounter>, java.io.Seriali
count = null;
}
- public void add(byte[] value) {
- add(value, 0, value.length);
- }
-
- public void add(byte[] value, int offset, int length) {
- if (value == null || length == 0) {
- return;
- }
-
- add(new String(value, offset, length));
- }
-
public void add(String value) {
if (value == null || value.isEmpty()) {
return;
@@ -235,29 +223,5 @@ public class BitmapCounter implements Comparable<BitmapCounter>, java.io.Seriali
else
return -1;
}
-
- private class ByteBufferBackedInputStream extends InputStream {
- private final ByteBuffer buffer;
-
- private ByteBufferBackedInputStream(ByteBuffer buf) {
- buffer = buf;
- }
-
- @Override
- public int read() throws IOException {
- return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
- }
-
- @Override
- public int read(byte[] bytes, int off, int len) throws IOException {
- if (!buffer.hasRemaining()) {
- return -1;
- }
-
- len = Math.min(len, buffer.remaining());
- buffer.get(bytes, off, len);
- return len;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
index a930db7..e216d0b 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapAggregatorTest.java
@@ -33,7 +33,7 @@ public class BitmapAggregatorTest {
BitmapCounter counter = new BitmapCounter();
counter.add(1);
counter.add(3333);
- counter.add("123".getBytes());
+ counter.add("123");
counter.add(123);
assertEquals(3, counter.getCount());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
index f7796c0..c9c1b51 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
@@ -35,7 +35,7 @@ public class BitmapCounterTest {
BitmapCounter counter = new BitmapCounter();
counter.add(1);
counter.add(3333);
- counter.add("123".getBytes());
+ counter.add("123");
counter.add(123);
assertEquals(3, counter.getCount());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
index 97b86ed..41efb2c 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapSerializerTest.java
@@ -47,7 +47,7 @@ public class BitmapSerializerTest extends LocalFileMetadataTestCase {
BitmapCounter counter = new BitmapCounter();
counter.add(1);
counter.add(3333);
- counter.add("123".getBytes());
+ counter.add("123");
counter.add(123);
assertEquals(3, counter.getCount());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
index e00ce16..6ff0d2f 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/TimedJsonStreamParser.java
@@ -31,9 +31,9 @@ import java.util.Arrays;
import com.fasterxml.jackson.databind.DeserializationFeature;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.ByteBufferBackedInputStream;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.kafka.util.ByteBufferBackedInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0aadb12/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
deleted file mode 100644
index 894a144..0000000
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/ByteBufferBackedInputStream.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.source.kafka.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-
-/**
- */
-public class ByteBufferBackedInputStream extends InputStream {
-
- private ByteBuffer buf;
-
- public ByteBufferBackedInputStream(ByteBuffer buf) {
- this.buf = buf;
- }
-
- @Override
- public int read() throws IOException {
- if (!buf.hasRemaining()) {
- return -1;
- }
- return buf.get() & 0xFF;
- }
-
- @Override
- public int read(byte[] bytes, int off, int len) throws IOException {
- if (!buf.hasRemaining()) {
- return -1;
- }
-
- len = Math.min(len, buf.remaining());
- buf.get(bytes, off, len);
- return len;
- }
-}
[34/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by li...@apache.org.
KYLIN-2331 by layer spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b7d87bb4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b7d87bb4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b7d87bb4
Branch: refs/heads/master-hbase1.x
Commit: b7d87bb4e6c4de484d910e9af04965392da518e5
Parents: 75625ae
Author: shaofengshi <sh...@apache.org>
Authored: Sun Dec 25 15:59:16 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 5 +-
.../main/config/assemblies/source-assembly.xml | 3 -
.../java/org/apache/kylin/job/DeployUtil.java | 1 -
build/conf/kylin-spark-conf.properties | 26 ++
build/conf/kylin.properties | 20 +
.../apache/kylin/common/KylinConfigBase.java | 64 ++-
.../kylin/common/persistence/ResourceStore.java | 35 ++
.../org/apache/kylin/common/util/Array.java | 2 +-
.../apache/kylin/common/util/SplittedBytes.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 2 +-
.../kylin/cube/common/RowKeySplitter.java | 15 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 27 +-
.../kylin/cube/cuboid/CuboidScheduler.java | 15 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +
.../kylin/cube/kv/AbstractRowKeyEncoder.java | 6 +-
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 7 +-
.../apache/kylin/cube/kv/RowKeyColumnIO.java | 2 +-
.../org/apache/kylin/cube/kv/RowKeyEncoder.java | 19 +-
.../kylin/cube/kv/RowKeyEncoderProvider.java | 2 +-
.../kylin/cube/model/AggregationGroup.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../cube/model/CubeJoinedFlatTableDesc.java | 2 +-
.../cube/model/CubeJoinedFlatTableEnrich.java | 6 +-
.../apache/kylin/cube/model/DictionaryDesc.java | 4 +-
.../apache/kylin/cube/model/DimensionDesc.java | 14 +-
.../kylin/cube/model/HBaseColumnDesc.java | 11 +-
.../kylin/cube/model/HBaseColumnFamilyDesc.java | 9 +-
.../kylin/cube/model/HBaseMappingDesc.java | 10 +-
.../apache/kylin/cube/model/HierarchyDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 2 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 18 +-
.../org/apache/kylin/cube/model/SelectRule.java | 2 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 41 +-
.../gridtable/AggregationCacheMemSizeTest.java | 6 +-
.../gridtable/AggregationCacheSpillTest.java | 5 +-
.../kylin/gridtable/SimpleGridTableTest.java | 11 +-
.../gridtable/SimpleInvertedIndexTest.java | 3 +-
.../metadata/measure/MeasureCodecTest.java | 6 +-
.../apache/kylin/dict/AppendTrieDictionary.java | 6 +-
.../org/apache/kylin/dict/NumberDictionary.java | 2 +-
.../apache/kylin/dict/NumberDictionary2.java | 2 +-
.../dict/NumberDictionaryForestBuilder.java | 4 +-
.../apache/kylin/dict/StringBytesConverter.java | 2 +-
.../kylin/job/execution/ExecutableManager.java | 21 +-
.../kylin/dimension/AbstractDateDimEnc.java | 12 +-
.../apache/kylin/dimension/BooleanDimEnc.java | 4 +-
.../kylin/dimension/DictionaryDimEnc.java | 21 +-
.../apache/kylin/dimension/FixedLenDimEnc.java | 4 +-
.../kylin/dimension/FixedLenHexDimEnc.java | 4 +-
.../org/apache/kylin/dimension/IntDimEnc.java | 4 +-
.../apache/kylin/dimension/IntegerDimEnc.java | 4 +-
.../kylin/dimension/OneMoreByteVLongDimEnc.java | 14 +-
.../kylin/measure/BufferedMeasureCodec.java | 10 +-
.../apache/kylin/measure/MeasureAggregator.java | 2 +
.../kylin/measure/MeasureAggregators.java | 18 +
.../org/apache/kylin/measure/MeasureCodec.java | 8 +-
.../apache/kylin/measure/MeasureIngester.java | 12 +-
.../org/apache/kylin/measure/MeasureType.java | 12 +-
.../measure/basic/BigDecimalMaxAggregator.java | 14 +
.../measure/basic/BigDecimalMinAggregator.java | 28 +-
.../measure/basic/BigDecimalSumAggregator.java | 10 +
.../kylin/measure/basic/DoubleIngester.java | 19 +-
.../measure/basic/DoubleMaxAggregator.java | 20 +-
.../measure/basic/DoubleMinAggregator.java | 20 +-
.../measure/basic/DoubleSumAggregator.java | 18 +-
.../kylin/measure/basic/LongIngester.java | 19 +-
.../kylin/measure/basic/LongMaxAggregator.java | 20 +-
.../kylin/measure/basic/LongMinAggregator.java | 20 +-
.../kylin/measure/basic/LongSumAggregator.java | 18 +-
.../kylin/measure/bitmap/BitmapAggregator.java | 13 +
.../kylin/measure/bitmap/BitmapCounter.java | 2 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 5 +
.../ExtendedColumnMeasureType.java | 15 +
.../kylin/measure/hllc/DenseRegister.java | 2 +-
.../kylin/measure/hllc/HLLCAggregator.java | 7 +
.../kylin/measure/hllc/HLLCMeasureType.java | 5 +
.../kylin/measure/hllc/HLLCSerializer.java | 11 +-
.../measure/hllc/HyperLogLogPlusTable.java | 2 +-
.../kylin/measure/hllc/SingleValueRegister.java | 2 +-
.../kylin/measure/hllc/SparseRegister.java | 2 +-
.../apache/kylin/measure/raw/RawAggregator.java | 14 +
.../apache/kylin/measure/raw/RawSerializer.java | 4 +-
.../measure/topn/DoubleDeltaSerializer.java | 7 +-
.../kylin/measure/topn/TopNAggregator.java | 9 +
.../apache/kylin/measure/topn/TopNCounter.java | 2 +-
.../kylin/measure/topn/TopNMeasureType.java | 4 +-
.../metadata/datatype/BigDecimalSerializer.java | 2 +-
.../metadata/datatype/BooleanSerializer.java | 36 +-
.../metadata/datatype/DataTypeSerializer.java | 17 +-
.../metadata/datatype/DateTimeSerializer.java | 30 +-
.../metadata/datatype/DoubleSerializer.java | 28 +-
.../kylin/metadata/datatype/Int4Serializer.java | 32 +-
.../metadata/datatype/Long8Serializer.java | 31 +-
.../kylin/metadata/datatype/LongSerializer.java | 32 +-
.../kylin/metadata/model/FunctionDesc.java | 22 +-
.../apache/kylin/metadata/model/JoinDesc.java | 7 +-
.../kylin/metadata/model/JoinTableDesc.java | 2 +-
.../apache/kylin/metadata/model/JoinsTree.java | 9 +-
.../kylin/metadata/model/MeasureDesc.java | 4 +-
.../metadata/model/ModelDimensionDesc.java | 2 +-
.../kylin/metadata/model/ParameterDesc.java | 13 +-
.../kylin/metadata/model/PartitionDesc.java | 4 +-
.../apache/kylin/metadata/model/TableRef.java | 14 +-
.../measure/AggregatorMemEstimateTest.java | 6 +-
.../storage/gtrecord/DictGridTableTest.java | 41 +-
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 7 +-
.../org/apache/kylin/engine/mr/HadoopUtil.java | 2 +-
.../kylin/engine/mr/JobBuilderSupport.java | 10 +
.../engine/mr/common/BaseCuboidBuilder.java | 173 ++++++++
.../kylin/engine/mr/common/CubeStatsReader.java | 5 +
.../kylin/engine/mr/common/NDCuboidBuilder.java | 96 +++++
.../engine/mr/steps/BaseCuboidMapperBase.java | 131 +-----
.../kylin/engine/mr/steps/NDCuboidMapper.java | 58 +--
.../kylin/engine/mr/steps/CubeReducerTest.java | 3 +-
engine-spark/pom.xml | 9 +
.../engine/spark/SparkBatchCubingEngine2.java | 33 ++
.../spark/SparkBatchCubingJobBuilder2.java | 85 ++++
.../apache/kylin/engine/spark/SparkCubing.java | 88 +++-
.../kylin/engine/spark/SparkCubingByLayer.java | 416 +++++++++++++++++++
.../kylin/engine/spark/SparkExecutable.java | 26 +-
.../spark/cube/DefaultTupleConverter.java | 2 +-
.../sandbox/kylin-spark-conf.properties | 28 ++
.../test_case_data/sandbox/kylin.properties | 10 +
pom.xml | 2 +-
.../kylin/rest/controller/CubeController.java | 8 +-
server/pom.xml | 7 +
.../apache/kylin/source/hive/HiveMRInput.java | 5 +-
.../hbase/steps/RowValueDecoderTest.java | 7 +-
tool/pom.xml | 4 +
129 files changed, 1715 insertions(+), 721 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 65e3a8c..873abff 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -47,6 +47,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-spark</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
@@ -173,7 +177,6 @@
<shadedClassifierName>job</shadedClassifierName>
<artifactSet>
<excludes>
- <exclude>io.netty:*</exclude>
<exclude>org.apache.zookeeper:*</exclude>
<exclude>net.sf.ehcache:*</exclude>
<exclude>org.apache.httpcomponents:*</exclude>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/assembly/src/main/config/assemblies/source-assembly.xml
----------------------------------------------------------------------
diff --git a/assembly/src/main/config/assemblies/source-assembly.xml b/assembly/src/main/config/assemblies/source-assembly.xml
index fad45aa..92584b5 100644
--- a/assembly/src/main/config/assemblies/source-assembly.xml
+++ b/assembly/src/main/config/assemblies/source-assembly.xml
@@ -97,9 +97,6 @@ limitations under the License.
<exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?docs(/.*)?]
</exclude>
- <!-- exclude unmaintained -->
- <exclude>%regex[(?!((?!${project.build.directory}/)))?engine-spark(/.*)?]
- </exclude>
</excludes>
</fileSet>
<!-- LICENSE, NOTICE, DEPENDENCIES, git.properties, etc. calculated at build time -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index a42b125..e8c7fae 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -82,7 +82,6 @@ public class DeployUtil {
config().overrideMRJobJarPath(jobJar.getAbsolutePath());
config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath());
- config().overrideSparkJobJarPath(getSparkJobJarFile().getAbsolutePath());
}
private static String getPomVersion() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/build/conf/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties
new file mode 100644
index 0000000..81567bb
--- /dev/null
+++ b/build/conf/kylin-spark-conf.properties
@@ -0,0 +1,26 @@
+spark.yarn.submit.file.replication=1
+spark.yarn.executor.memoryOverhead=200
+spark.yarn.driver.memoryOverhead=384
+spark.master=yarn
+spark.submit.deployMode=cluster
+spark.eventLog.enabled=true
+spark.yarn.scheduler.heartbeat.interval-ms=5000
+spark.yarn.preserve.staging.files=true
+spark.yarn.queue=default
+spark.yarn.containerLauncherMaxThreads=25
+spark.yarn.max.executor.failures=3
+spark.eventLog.dir=hdfs\:///kylin/spark-history
+spark.history.kerberos.enabled=true
+spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
+spark.history.ui.port=18080
+spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+spark.executor.memory=4G
+spark.storage.memoryFraction=0.3
+spark.executor.cores=4
+spark.executor.instances=8
+spark.history.kerberos.keytab=none
+spark.history.kerberos.principal=none
+#spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+spark.driver.extraJavaOptions=-Dhdp.version=current
+spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 5e7d235..98b66cb 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -128,6 +128,26 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
+### Spark Engine ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
+kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+
+# Spark install home, default be $KYLIN_HOME/spark/
+#kylin.engine.spark.spark-home=
+
+# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
+#kylin.engine.spark.properties-file=
+
+# Estimate the RDD partition numbers
+kylin.engine.spark.rdd-partition-cut-mb=200
+
+# Minimal partition numbers of rdd
+kylin.engine.spark.min-partition=1
+
+# Max partition numbers of rdd
+kylin.engine.spark.max-partition=500
+
### CUBE | DICTIONARY ###
# 'auto', 'inmem', 'layer' or 'random' for testing
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 88c4dbc..77b1e1c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -652,6 +652,7 @@ abstract public class KylinConfigBase implements Serializable {
// ref constants in IEngineAware
r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine");
r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2");
+ r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2");
return r;
}
@@ -728,29 +729,62 @@ abstract public class KylinConfigBase implements Serializable {
// ENGINE.SPARK
// ============================================================================
- public String getKylinSparkJobJarPath() {
- final String jobJar = getOptional("kylin.engine.spark.job-jar");
- if (StringUtils.isNotEmpty(jobJar)) {
- return jobJar;
+ public String getSparkHome() {
+ String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark");
+ File f = new File(sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ } else {
+ String home = getKylinHome();
+ f = new File(home, sparkHome);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ }
}
- String kylinHome = getKylinHome();
- if (StringUtils.isEmpty(kylinHome)) {
- return "";
+
+ throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties");
+
+ }
+
+ public String getSparkHadoopConfDir() {
+ return getRequired("kylin.engine.spark.env.hadoop-conf-dir");
+ }
+
+ public String getSparkConfFile() {
+ String conf = getOptional("kylin.engine.spark.properties-file", "conf/kylin-spark-conf.properties");
+ File f = new File(conf);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ } else {
+ String home = getKylinHome();
+ f = new File(home, conf);
+ if (f.exists()) {
+ return f.getAbsolutePath();
+ }
}
- return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN);
+
+ throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist.");
}
- public void overrideSparkJobJarPath(String path) {
- logger.info("override " + "kylin.engine.spark.job-jar" + " to " + path);
- System.setProperty("kylin.engine.spark.job-jar", path);
+ public String getSparkAdditionalJars() {
+ return getOptional("kylin.engine.spark.additional-jars", "");
}
- public String getSparkHome() {
- return getRequired("kylin.engine.spark.spark-home");
+ public float getSparkRDDPartitionCutMB() {
+ return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "200.0"));
+ }
+
+
+ public int getSparkMinPartition() {
+ return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", "1"));
+ }
+
+ public int getSparkMaxPartition() {
+ return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "500"));
}
- public String getSparkMaster() {
- return getRequired("kylin.engine.spark.spark-master");
+ public boolean isSparkSanityCheckEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.spark.sanity-check-enabled", "false"));
}
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 0580576..9549569 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -21,17 +21,21 @@ package org.apache.kylin.common.persistence;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OptionsHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -311,4 +315,35 @@ abstract public class ResourceStore {
return collector;
}
+ public static String dumpResources(KylinConfig kylinConfig, Collection<String> dumpList) throws IOException {
+ File tmp = File.createTempFile("kylin_job_meta", "");
+ FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
+
+ File metaDir = new File(tmp, "meta");
+ metaDir.mkdirs();
+
+ // write kylin.properties
+ File kylinPropsFile = new File(metaDir, "kylin.properties");
+ kylinConfig.writeProperties(kylinPropsFile);
+
+ ResourceStore from = ResourceStore.getStore(kylinConfig);
+ KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath());
+ ResourceStore to = ResourceStore.getStore(localConfig);
+ for (String path : dumpList) {
+ RawResource res = from.getResource(path);
+ if (res == null)
+ throw new IllegalStateException("No resource found at -- " + path);
+ to.putResource(path, res.inputStream, res.timestamp);
+ res.inputStream.close();
+ }
+
+ String metaDirURI = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath());
+ if (metaDirURI.startsWith("/")) // note Path on windows is like "d:/../..."
+ metaDirURI = "file://" + metaDirURI;
+ else
+ metaDirURI = "file:///" + metaDirURI;
+ logger.info("meta dir is: " + metaDirURI);
+
+ return metaDirURI;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-common/src/main/java/org/apache/kylin/common/util/Array.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Array.java b/core-common/src/main/java/org/apache/kylin/common/util/Array.java
index 7447b46..b25b764 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Array.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Array.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
/*
* An array with correct equals(), hashCode(), compareTo() and toString()
*/
-public class Array<T> implements Comparable<Array<T>> {
+public class Array<T> implements Comparable<Array<T>>, java.io.Serializable {
public T[] data;
public Array(T[] data) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
index 8751b78..ae380cb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java
@@ -22,7 +22,7 @@ package org.apache.kylin.common.util;
* @author George Song (ysong1)
*
*/
-public class SplittedBytes {
+public class SplittedBytes implements java.io.Serializable {
public SplittedBytes(int length) {
this.value = new byte[length];
this.length = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 36a6044..45310f0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -51,7 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
+public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment, java.io.Serializable {
@JsonBackReference
private CubeInstance cubeInstance;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
index 67f1751..acebce4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
@@ -26,9 +26,11 @@ import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.RowKeyColDesc;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.metadata.model.TblColRef;
-public class RowKeySplitter {
+public class RowKeySplitter implements java.io.Serializable {
private CubeDesc cubeDesc;
private RowKeyColumnIO colIO;
@@ -37,7 +39,6 @@ public class RowKeySplitter {
private int[] splitOffsets;
private int bufferSize;
- private long lastSplittedCuboidId;
private boolean enableSharding;
private short shardId;
@@ -64,7 +65,13 @@ public class RowKeySplitter {
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
this.enableSharding = cubeSeg.isEnableSharding();
this.cubeDesc = cubeSeg.getCubeDesc();
- this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg));
+ IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);
+
+ for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
+ dimEncoding.get(rowKeyColDesc.getColRef());
+ }
+
+ this.colIO = new RowKeyColumnIO(dimEncoding);
this.splitBuffers = new SplittedBytes[splitLen];
this.splitOffsets = new int[splitLen];
@@ -105,7 +112,7 @@ public class RowKeySplitter {
System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
offset += RowConstants.ROWKEY_CUBOIDID_LEN;
- lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
+ long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId);
// rowkey columns
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
index 7503fbf..dd22d6a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java
@@ -18,15 +18,10 @@
package org.apache.kylin.cube.cuboid;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
@@ -37,12 +32,16 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.ComparisonChain;
-import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
-public class Cuboid implements Comparable<Cuboid> {
+public class Cuboid implements Comparable<Cuboid>, java.io.Serializable {
private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index 733aded..ffb0a5e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -21,6 +21,12 @@ package org.apache.kylin.cube.cuboid;
/**
*/
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.CubeDesc;
+
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -28,14 +34,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.kylin.cube.model.AggregationGroup;
-import org.apache.kylin.cube.model.CubeDesc;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class CuboidScheduler {
+public class CuboidScheduler implements java.io.Serializable {
private final CubeDesc cubeDesc;
private final long max;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 85a694f..398c594 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -463,6 +463,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
for (int i = 0; i < totalSum.length; i++) {
if (totalSum[i] instanceof DoubleMutable) {
totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get());
+ } else if (totalSum[i] instanceof Double) {
+ totalSum[i] = Math.round(((Double) totalSum[i]).doubleValue());
} else if (totalSum[i] instanceof TopNCounter) {
TopNCounter counter = (TopNCounter) totalSum[i];
Iterator<Counter> iterator = counter.iterator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
index bfe6eb4..2becde4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.kv;
-import java.util.Map;
-
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
@@ -30,12 +28,14 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Map;
+
/**
*
* @author xjiang
*
*/
-public abstract class AbstractRowKeyEncoder {
+public abstract class AbstractRowKeyEncoder implements java.io.Serializable {
protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class);
public static final byte DEFAULT_BLANK_BYTE = DimensionEncoding.NULL;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index a4d2d6f..bd9554a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -18,8 +18,7 @@
package org.apache.kylin.cube.kv;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
@@ -33,9 +32,9 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
+import java.util.Map;
-public class CubeDimEncMap implements IDimensionEncodingMap {
+public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable {
private static final Logger logger = LoggerFactory.getLogger(CubeDimEncMap.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
index fbb93db..65911a0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java
@@ -29,7 +29,7 @@ import org.apache.kylin.metadata.model.TblColRef;
*
* @author yangli9
*/
-public class RowKeyColumnIO {
+public class RowKeyColumnIO implements java.io.Serializable {
//private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
index bf20de1..a669fb1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java
@@ -18,11 +18,7 @@
package org.apache.kylin.cube.kv;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.base.Preconditions;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -32,9 +28,12 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
-public class RowKeyEncoder extends AbstractRowKeyEncoder {
+public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable {
private int bodyLength = 0;
private RowKeyColumnIO colIO;
@@ -42,11 +41,13 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
protected boolean enableSharding;
private int uhcOffset = -1;//it's a offset to the beginning of body
private int uhcLength = -1;
+ private int headerLength;
public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) {
super(cubeSeg, cuboid);
enableSharding = cubeSeg.isEnableSharding();
- Set<TblColRef> shardByColumns = cubeSeg.getShardByColumns();
+ headerLength = cubeSeg.getRowKeyPreambleSize();
+ Set<TblColRef> shardByColumns = cubeSeg.getCubeDesc().getShardByColumns();
if (shardByColumns.size() > 1) {
throw new IllegalStateException("Does not support multiple UHC now");
}
@@ -61,7 +62,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder {
}
public int getHeaderLength() {
- return cubeSeg.getRowKeyPreambleSize();
+ return headerLength;
}
public int getBytesLength() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
index 2b1dea7..791b564 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java
@@ -25,7 +25,7 @@ import org.apache.kylin.cube.cuboid.Cuboid;
import com.google.common.collect.Maps;
-public class RowKeyEncoderProvider {
+public class RowKeyEncoderProvider implements java.io.Serializable {
private CubeSegment cubeSegment;
private HashMap<Long, RowKeyEncoder> rowKeyEncoders;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 1d4d496..bc9fa06 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -37,7 +37,7 @@ import com.google.common.collect.Lists;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class AggregationGroup implements Serializable{
- public static class HierarchyMask {
+ public static class HierarchyMask implements java.io.Serializable {
public long fullMask; // 00000111
public long[] allMasks; // 00000100,00000110,00000111
public long[] dims; // 00000100,00000010,00000001
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 556cc72..7e599da 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -100,7 +100,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
LOOKUP, PK_FK, EXTENDED_COLUMN
}
- public static class DeriveInfo {
+ public static class DeriveInfo implements java.io.Serializable {
public DeriveType type;
public JoinDesc join;
public TblColRef[] columns;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 94e1a7c..2a68116 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -36,7 +36,7 @@ import com.google.common.collect.Maps;
/**
*/
-public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
+public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Serializable {
protected final String tableName;
protected final CubeDesc cubeDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index a1312b5..e829aeb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube.model;
-import java.util.List;
-
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -28,10 +26,12 @@ import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.List;
+
/**
* An enrich of IJoinedFlatTableDesc for cubes
*/
-public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc {
+public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, java.io.Serializable {
private CubeDesc cubeDesc;
private IJoinedFlatTableDesc flatDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
index fb7053e..ca2183a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
@@ -26,10 +26,8 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import java.io.Serializable;
-
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DictionaryDesc implements Serializable{
+public class DictionaryDesc implements java.io.Serializable {
@JsonProperty("column")
private String column;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index 29fd6c3..3488c36 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -18,10 +18,8 @@
package org.apache.kylin.cube.model;
-import java.io.Serializable;
import java.util.Arrays;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -36,7 +34,7 @@ import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc implements Serializable {
+public class DimensionDesc implements java.io.Serializable {
@JsonProperty("name")
private String name;
@@ -140,16 +138,6 @@ public class DimensionDesc implements Serializable {
}
@Override
- public boolean equals(Object o) {
- throw new NotImplementedException();
- }
-
- @Override
- public int hashCode() {
- throw new NotImplementedException();
- }
-
- @Override
public String toString() {
return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
index fb491f8..7007342 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java
@@ -18,19 +18,18 @@
package org.apache.kylin.cube.model;
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import java.util.Arrays;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnDesc {
+public class HBaseColumnDesc implements java.io.Serializable {
@JsonProperty("qualifier")
private String qualifier;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
index c5b2e19..85c2c17 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java
@@ -18,18 +18,17 @@
package org.apache.kylin.cube.model;
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.MeasureDesc;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
+import java.util.Arrays;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnFamilyDesc {
+public class HBaseColumnFamilyDesc implements java.io.Serializable {
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index 84d87de..d1e5829 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -18,23 +18,21 @@
package org.apache.kylin.cube.model;
-import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseMappingDesc implements Serializable{
+public class HBaseMappingDesc implements java.io.Serializable {
@JsonProperty("column_family")
private HBaseColumnFamilyDesc[] columnFamily;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
index f88d4d2..b72f220 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java
@@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HierarchyDesc {
+public class HierarchyDesc implements java.io.Serializable {
@JsonProperty("level")
private String level;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index ef34a9b..71fd4b9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -40,7 +40,7 @@ import com.google.common.base.Preconditions;
*
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyColDesc {
+public class RowKeyColDesc implements java.io.Serializable {
@JsonProperty("column")
private String column;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index 1bd06d2..00557c5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,25 +18,23 @@
package org.apache.kylin.cube.model;
-import java.io.Serializable;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.kylin.metadata.model.TblColRef;
+
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyDesc implements Serializable{
+public class RowKeyDesc implements java.io.Serializable {
@JsonProperty("rowkey_columns")
private RowKeyColDesc[] rowkeyColumns;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
index 63b0fc4..4a6c510 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java
@@ -20,7 +20,7 @@ package org.apache.kylin.cube.model;
import com.fasterxml.jackson.annotation.JsonProperty;
-public class SelectRule {
+public class SelectRule implements java.io.Serializable {
@JsonProperty("hierarchy_dims")
public String[][] hierarchy_dims;
@JsonProperty("mandatory_dims")
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index b8d116c..6190f29 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -28,7 +28,6 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
public class UnitTestSupport {
@@ -84,16 +83,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5")));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5")));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5")));
}
return result;
}
@@ -106,16 +105,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 3807b70..b164e54 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -36,8 +36,6 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.Test;
@@ -100,10 +98,10 @@ public class AggregationCacheMemSizeTest {
private MeasureAggregator<?>[] createNoMemHungryAggrs() {
LongSumAggregator longSum = new LongSumAggregator();
- longSum.aggregate(new LongMutable(10));
+ longSum.aggregate(new Long(10));
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
- doubleSum.aggregate(new DoubleMutable(10));
+ doubleSum.aggregate(new Double(10));
BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator();
decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index d7787e0..74c2659 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -28,7 +28,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -91,7 +90,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(20, ((LongMutable) returnRecord[3]).get());
+ assertEquals(20, ((Long) returnRecord[3]).longValue());
assertEquals(21, ((BigDecimal) returnRecord[4]).longValue());
count++;
@@ -133,7 +132,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
for (GTRecord record : scanner) {
assertNotNull(record);
Object[] returnRecord = record.getValues();
- assertEquals(80000, ((LongMutable) returnRecord[3]).get());
+ assertEquals(80000, ((Long) returnRecord[3]).longValue());
assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue());
count++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
index f315b1e..4ac6644 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java
@@ -29,7 +29,6 @@ import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -97,7 +96,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
assertTrue(((String) v[0]).startsWith("2015-"));
assertTrue(((String) v[2]).equals("Food"));
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
System.out.println(r);
}
@@ -114,19 +113,19 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase {
Object[] v = r.getValues();
switch (i) {
case 0:
- assertTrue(((LongMutable) v[3]).get() == 20);
+ assertTrue(((Long) v[3]).longValue() == 20);
assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0);
break;
case 1:
- assertTrue(((LongMutable) v[3]).get() == 30);
+ assertTrue(((Long) v[3]).longValue() == 30);
assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5);
break;
case 2:
- assertTrue(((LongMutable) v[3]).get() == 40);
+ assertTrue(((Long) v[3]).longValue() == 40);
assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0);
break;
case 3:
- assertTrue(((LongMutable) v[3]).get() == 10);
+ assertTrue(((Long) v[3]).longValue() == 10);
assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5);
break;
default:
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
index 2cf58ac..5e3e771 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
@@ -27,7 +27,6 @@ import java.util.ArrayList;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.datatype.StringSerializer;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -72,7 +71,7 @@ public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
GTRowBlock.Writer writer = mockBlock.getWriter();
GTRecord record = new GTRecord(info);
for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0));
+ record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
for (int j = 0; j < info.getRowBlockSize(); j++) {
writer.append(record);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index fc20418..ad4d90b 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -27,8 +27,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.AfterClass;
@@ -54,8 +52,8 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") };
BufferedMeasureCodec codec = new BufferedMeasureCodec(descs);
- DoubleMutable d = new DoubleMutable(1.0);
- LongMutable l = new LongMutable(2);
+ Double d = new Double(1.0);
+ Long l = new Long(2);
BigDecimal b = new BigDecimal("333.1234");
HLLCounter hllc = new HLLCounter(16);
hllc.add("1234567");
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 503c29e..5029bc4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -133,7 +133,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
}
- public static class DictSliceKey implements WritableComparable {
+ public static class DictSliceKey implements WritableComparable, java.io.Serializable {
byte[] key;
public static DictSliceKey wrap(byte[] key) {
@@ -174,7 +174,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
}
- public static class DictSlice<T> implements Writable {
+ public static class DictSlice<T> implements Writable, java.io.Serializable {
public DictSlice() {
}
@@ -438,7 +438,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
}
- public static class DictNode implements Writable {
+ public static class DictNode implements Writable, java.io.Serializable {
public byte[] part;
public int id = -1;
public boolean isEndOfValue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index f1b1b3d..9458e9d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -153,7 +153,7 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
}
}
- static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
+ static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
index b5ce1ff..80e9940 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java
@@ -24,7 +24,7 @@ package org.apache.kylin.dict;
@SuppressWarnings("serial")
public class NumberDictionary2<T> extends NumberDictionary<T> {
- static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
+ static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>();
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
index 5502a74..7c750f4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
@@ -26,10 +26,10 @@ import org.apache.kylin.dict.NumberDictionary.NumberBytesCodec;
*/
public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<String> {
- public static class Number2BytesConverter implements BytesConverter<String> {
+ public static class Number2BytesConverter implements BytesConverter<String>, java.io.Serializable {
static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT;
- static final ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>();
+ static final transient ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>();
static NumberBytesCodec getCodec() {
NumberBytesCodec codec = LOCAL.get();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
index 0bec6a1..9107a4c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java
@@ -20,7 +20,7 @@ package org.apache.kylin.dict;
import org.apache.kylin.common.util.Bytes;
-public class StringBytesConverter implements BytesConverter<String> {
+public class StringBytesConverter implements BytesConverter<String>, java.io.Serializable {
@Override
public byte[] convertToBytes(String v) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0273fd8..466cdad 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -18,12 +18,9 @@
package org.apache.kylin.job.execution;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.job.dao.ExecutableDao;
@@ -34,9 +31,11 @@ import org.apache.kylin.job.exception.PersistentException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
/**
*/
@@ -352,6 +351,10 @@ public class ExecutableManager {
}
}
}
+
+ if (job.getStatus() == ExecutableState.SUCCEED) {
+ updateJobOutput(job.getId(), ExecutableState.READY, null, null);
+ }
}
public void pauseJob(String jobId) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
index ec6347f..a54bcda 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
@@ -18,6 +18,10 @@
package org.apache.kylin.dimension;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -25,10 +29,6 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-
public class AbstractDateDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
@@ -81,11 +81,9 @@ public class AbstractDateDimEnc extends DimensionEncoding {
@Override
public DataTypeSerializer<Object> asDataTypeSerializer() {
return new DataTypeSerializer<Object>() {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index 08fbe51..12162b2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -109,11 +109,9 @@ public class BooleanDimEnc extends DimensionEncoding {
}
private class BooleanSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 500b410..48238dc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -39,12 +39,15 @@ public class DictionaryDimEnc extends DimensionEncoding {
// ============================================================================
// could use a lazy loading trick here, to prevent loading all dictionaries of a segment at once
- private final Dictionary<String> dict;
- private final int fixedLen;
+ private Dictionary<String> dict;
+ private int fixedLen;
// used in encode(), when a value does not exist in dictionary
- private final int roundingFlag;
- private final byte defaultByte;
+ private int roundingFlag;
+ private byte defaultByte;
+
+ public DictionaryDimEnc() {
+ }
public DictionaryDimEnc(Dictionary<String> dict) {
this(dict, 0, NULL);
@@ -145,12 +148,18 @@ public class DictionaryDimEnc extends DimensionEncoding {
@Override
public void writeExternal(ObjectOutput out) throws IOException {
- throw new UnsupportedOperationException();
+ out.writeInt(fixedLen);
+ out.writeInt(roundingFlag);
+ out.write(defaultByte);
+ out.writeObject(dict);
}
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- throw new UnsupportedOperationException();
+ this.fixedLen = in.readInt();
+ this.roundingFlag = in.readInt();
+ this.defaultByte = in.readByte();
+ this.dict = (Dictionary<String>) in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index b219766..f7f02a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -129,11 +129,9 @@ public class FixedLenDimEnc extends DimensionEncoding {
}
public class FixedLenSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
index 83118fc..f90a40e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
@@ -224,11 +224,9 @@ public class FixedLenHexDimEnc extends DimensionEncoding {
}
public class FixedLenSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[bytelen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index f25f2a6..aa954da 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -109,11 +109,9 @@ public class IntDimEnc extends DimensionEncoding {
}
public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 090dc83..0875a7f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -135,11 +135,9 @@ public class IntegerDimEnc extends DimensionEncoding {
}
public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[fixedLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
index 993aac3..c0c52d1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
@@ -18,17 +18,17 @@
package org.apache.kylin.dimension;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* not being used yet, prepared for future
*/
@@ -118,11 +118,9 @@ public class OneMoreByteVLongDimEnc extends DimensionEncoding {
}
public class VLongSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
private byte[] currentBuf() {
- byte[] buf = current.get();
+ byte[] buf = (byte[]) current.get();
if (buf == null) {
buf = new byte[byteLen];
current.set(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
index 8a5481c..44e5708 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java
@@ -18,26 +18,26 @@
package org.apache.kylin.measure;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Collection;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
/**
* This class embeds a reusable byte buffer for measure encoding, and is not thread-safe.
* The buffer will grow to accommodate BufferOverflowException until a limit.
* The problem here to solve is some measure type cannot provide accurate DataTypeSerializer.maxLength()
*/
@SuppressWarnings({ "unchecked" })
-public class BufferedMeasureCodec {
+public class BufferedMeasureCodec implements java.io.Serializable {
public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; // 1 MB
public static final int MAX_BUFFER_SIZE = 1 * 1024 * DEFAULT_BUFFER_SIZE; // 1 GB
final private MeasureCodec codec;
- private ByteBuffer buf;
+ private transient ByteBuffer buf;
final private int[] measureSizes;
public BufferedMeasureCodec(Collection<MeasureDesc> measureDescs) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
index 7b74225..929eff0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java
@@ -71,6 +71,8 @@ abstract public class MeasureAggregator<V> implements Serializable {
abstract public void aggregate(V value);
+ abstract public V aggregate(V value1, V value2);
+
abstract public V getState();
// get an estimate of memory consumption UPPER BOUND
[46/50] [abbrv] kylin git commit: minor,
restore @Ignore to ITKylinQueryTest
Posted by li...@apache.org.
minor, restore @Ignore to ITKylinQueryTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c9ab289c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c9ab289c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c9ab289c
Branch: refs/heads/master-hbase1.x
Commit: c9ab289c09bf95b97d67315f5d7e0b9d634aa48b
Parents: be11dc6
Author: Yang Li <li...@apache.org>
Authored: Wed Jan 11 21:11:47 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Jan 11 21:11:47 2017 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/query/ITKylinQueryTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c9ab289c/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index aaf0925..82d0873 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -47,7 +47,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
[48/50] [abbrv] kylin git commit: KYLIN-2382 fix column order for
select *
Posted by li...@apache.org.
KYLIN-2382 fix column order for select *
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b0a406ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b0a406ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b0a406ff
Branch: refs/heads/master-hbase1.x
Commit: b0a406ff3c43d0cf6f8aa0b632e324d933440439
Parents: 0603099
Author: Yang Li <li...@apache.org>
Authored: Thu Jan 12 08:08:29 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Jan 12 08:08:29 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/query/schema/OLAPTable.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b0a406ff/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index fe6a806..9d20fc7 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -19,6 +19,8 @@
package org.apache.kylin.query.schema;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -218,6 +220,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
logger.info("To avoid overflow, upgraded {}'s type from {} to {}", tableColumns.get(index), tableColumns.get(index).getType(), tableColumns.get(index).getUpgradedType());
}
+ Collections.sort(tableColumns, new Comparator<ColumnDesc>() {
+ @Override
+ public int compare(ColumnDesc o1, ColumnDesc o2) {
+ return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
+ }
+ });
return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
}
[09/50] [abbrv] kylin git commit: KYLIN-2360 fix minor SonarCube issue
Posted by li...@apache.org.
KYLIN-2360 fix minor SonarCube issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/503d232d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/503d232d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/503d232d
Branch: refs/heads/master-hbase1.x
Commit: 503d232dc1875d504959a3e00da2eb62a7b7d599
Parents: 980b03b
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 18:04:56 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 18:09:45 2017 +0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableWindow.java | 53 +++++++++++++++++++-
.../kylin/common/persistence/ResourceTool.java | 5 +-
.../apache/kylin/common/util/StringUtil.java | 2 +-
.../org/apache/kylin/dict/CacheDictionary.java | 29 ++++++-----
4 files changed, 71 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 547210c..216b07c 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -80,6 +80,7 @@ public class EnumerableWindow extends Window implements EnumerableRel {
constants, rowType, groups);
}
+ @Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
return super.computeSelfCost(planner, mq)
.multiplyBy(EnumerableConvention.COST_MULTIPLIER);
@@ -104,6 +105,7 @@ public class EnumerableWindow extends Window implements EnumerableRel {
this.constants = constants;
}
+ @Override
public Expression field(BlockBuilder list, int index, Type storageType) {
if (index < actualInputFieldCount) {
Expression current = list.append("current", row);
@@ -113,6 +115,55 @@ public class EnumerableWindow extends Window implements EnumerableRel {
}
}
+ private void sampleOfTheGeneratedWindowedAggregate() {
+ // Here's overview of the generated code
+ // For each list of rows that have the same partitioning key, evaluate
+ // all of the windowed aggregate functions.
+
+ // builder
+ Iterator<Integer[]> iterator = null;
+
+ // builder3
+ Integer[] rows = iterator.next();
+
+ int prevStart = -1;
+ int prevEnd = -1;
+
+ for (int i = 0; i < rows.length; i++) {
+ // builder4
+ Integer row = rows[i];
+
+ int start = 0;
+ int end = 100;
+ if (start != prevStart || end != prevEnd) {
+ // builder5
+ int actualStart = 0;
+ if (start != prevStart || end < prevEnd) {
+ // builder6
+ // recompute
+ actualStart = start;
+ // implementReset
+ } else { // must be start == prevStart && end > prevEnd
+ actualStart = prevEnd + 1;
+ }
+ prevStart = start;
+ prevEnd = end;
+
+ if (start != -1) {
+ for (int j = actualStart; j <= end; j++) {
+ // builder7
+ // implementAdd
+ }
+ }
+ // implementResult
+ // list.add(new Xxx(row.deptno, row.empid, sum, count));
+ }
+ }
+ // multiMap.clear(); // allows gc
+ // source = Linq4j.asEnumerable(list);
+ }
+
+ @Override
public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
final JavaTypeFactory typeFactory = implementor.getTypeFactory();
final EnumerableRel child = (EnumerableRel) getInput();
@@ -121,7 +172,7 @@ public class EnumerableWindow extends Window implements EnumerableRel {
Expression source_ = builder.append("source", result.block);
final List<Expression> translatedConstants =
- new ArrayList<Expression>(constants.size());
+ new ArrayList<>(constants.size());
for (RexLiteral constant : constants) {
translatedConstants.add(
RexToLixTranslator.translateLiteral(constant, constant.getType(),
http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 2c8bc83..b3aac09 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -28,11 +28,14 @@ import java.util.NavigableSet;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ResourceTool {
private static String[] includes = null;
private static String[] excludes = null;
+ private static final Logger logger = LoggerFactory.getLogger(ResourceTool.class);
public static void main(String[] args) throws IOException {
args = StringUtil.filterSystemArgs(args);
@@ -164,7 +167,7 @@ public class ResourceTool {
}
} catch (Exception ex) {
System.err.println("Failed to open " + path);
- ex.printStackTrace();
+ logger.error(ex.getLocalizedMessage(), ex);
}
}
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 12e7279..96d294b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -161,7 +161,7 @@ public class StringUtil {
if (!s.isEmpty())
r.add(s);
}
- return (String[]) r.toArray(new String[r.size()]);
+ return r.toArray(new String[r.size()]);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/503d232d/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index 1b3bfa1..1e260b2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -25,20 +25,19 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
- * Created by xiefan on 16-12-30.
*/
-abstract public class CacheDictionary<T> extends Dictionary<T> {
+public abstract class CacheDictionary<T> extends Dictionary<T> {
private static final long serialVersionUID = 1L;
- transient protected boolean enableValueCache = false;
+ protected transient boolean enableValueCache = false;
- transient private SoftReference<Map> valueToIdCache;
+ private transient SoftReference<ConcurrentHashMap> valueToIdCache;
- transient private SoftReference<Object[]> idToValueCache;
+ private transient SoftReference<Object[]> idToValueCache;
- transient protected int baseId;
+ protected transient int baseId;
- transient protected BytesConverter<T> bytesConvert;
+ protected transient BytesConverter<T> bytesConvert;
public CacheDictionary() {
@@ -46,11 +45,11 @@ abstract public class CacheDictionary<T> extends Dictionary<T> {
//value --> id
@Override
- final protected int getIdFromValueImpl(T value, int roundingFlag) {
+ protected final int getIdFromValueImpl(T value, int roundingFlag) {
if (enableValueCache && roundingFlag == 0) {
Map cache = valueToIdCache.get(); // SoftReference to skip cache gracefully when short of memory
if (cache != null) {
- Integer id = null;
+ Integer id;
id = (Integer) cache.get(value);
if (id != null)
return id.intValue();
@@ -66,7 +65,7 @@ abstract public class CacheDictionary<T> extends Dictionary<T> {
//id --> value
@Override
- final protected T getValueFromIdImpl(int id) {
+ protected final T getValueFromIdImpl(int id) {
if (enableValueCache) {
Object[] cache = idToValueCache.get();
if (cache != null) {
@@ -83,7 +82,7 @@ abstract public class CacheDictionary<T> extends Dictionary<T> {
return bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length);
}
- final protected int calcSeqNoFromId(int id) {
+ protected final int calcSeqNoFromId(int id) {
int seq = id - baseId;
if (seq < 0 || seq >= getSize()) {
throw new IllegalArgumentException("Not a valid ID: " + id);
@@ -91,15 +90,15 @@ abstract public class CacheDictionary<T> extends Dictionary<T> {
return seq;
}
- final public void enableCache() {
+ public final void enableCache() {
this.enableValueCache = true;
if (this.valueToIdCache == null)
- this.valueToIdCache = new SoftReference<Map>(new ConcurrentHashMap());
+ this.valueToIdCache = new SoftReference<>(new ConcurrentHashMap());
if (this.idToValueCache == null)
- this.idToValueCache = new SoftReference<Object[]>(new Object[getSize()]);
+ this.idToValueCache = new SoftReference<>(new Object[getSize()]);
}
- final public void disableCache() {
+ public final void disableCache() {
this.enableValueCache = false;
this.valueToIdCache = null;
this.idToValueCache = null;
[24/50] [abbrv] kylin git commit: KYLIN-2353 minor code review
Posted by li...@apache.org.
KYLIN-2353 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56daf6d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56daf6d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56daf6d5
Branch: refs/heads/master-hbase1.x
Commit: 56daf6d592ae454b3bca97c4a7a948781485befa
Parents: 731a53a
Author: Yang Li <li...@apache.org>
Authored: Sat Jan 7 19:30:27 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:30:27 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/measure/bitmap/BitmapCounter.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56daf6d5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a18ac4a..a632b0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -51,7 +51,9 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
if (buffer != null) {
+ @SuppressWarnings("unused")
int version = buffer.getInt();
+ @SuppressWarnings("unused")
int size = buffer.getInt();
count = buffer.getInt();
@@ -79,7 +81,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
public void add(int value) {
getBitmap().add(value);
- count = getBitmap().getCardinality();
+ count = null;
}
public void add(byte[] value) {
@@ -103,12 +105,12 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
public void merge(BitmapCounter another) {
getBitmap().or(another.getBitmap());
- count = getBitmap().getCardinality();
+ count = null;
}
public void intersect(BitmapCounter another) {
getBitmap().and(another.getBitmap());
- count = getBitmap().getCardinality();
+ count = null;
}
public int getCount() {
@@ -148,7 +150,6 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
// keep forward compatibility
if (version == VERSION) {
- @SuppressWarnings("unused")
int size = in.getInt();
count = in.getInt();
in.position(mark);
@@ -258,4 +259,5 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return len;
}
}
+
}
[47/50] [abbrv] kylin git commit: fix query CI
Posted by li...@apache.org.
fix query CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/06030999
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/06030999
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/06030999
Branch: refs/heads/master-hbase1.x
Commit: 060309991f56988e097666346c5c13be7d3ba043
Parents: c9ab289
Author: Yang Li <li...@apache.org>
Authored: Wed Jan 11 21:35:50 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Jan 11 21:35:50 2017 +0800
----------------------------------------------------------------------
kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/06030999/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index a8eb505..deb1104 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -234,7 +234,6 @@ public class KylinTestBase {
for (int i = 0; i < columnNames.length; i++) {
columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName();
}
- Arrays.sort(columnNames);
queryTable = new SortedTable(queryTable, columnNames);
}
[29/50] [abbrv] kylin git commit: fix NPE in CacheDictionary in Spark
cubing
Posted by li...@apache.org.
fix NPE in CacheDictionary in Spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/66bca9a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/66bca9a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/66bca9a6
Branch: refs/heads/master-hbase1.x
Commit: 66bca9a676fda02aa88cc25bad2e545f779f6dde
Parents: 2cf52b4
Author: shaofengshi <sh...@apache.org>
Authored: Sat Jan 7 10:17:33 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/CacheDictionary.java | 2 +-
.../apache/kylin/engine/mr/BatchCubingJobBuilder.java | 5 ++---
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 7 +++----
.../org/apache/kylin/engine/mr/JobBuilderSupport.java | 14 +-------------
.../engine/spark/SparkBatchCubingJobBuilder2.java | 1 +
.../apache/kylin/engine/spark/SparkCubingByLayer.java | 2 +-
.../kylin/engine/spark/SparkCubingJobBuilder.java | 1 -
7 files changed, 9 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index d7ed6bd..b2bad53 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -35,7 +35,7 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
protected transient int baseId;
- protected transient BytesConverter<T> bytesConvert;
+ protected BytesConverter<T> bytesConvert;
public CacheDictionary() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 456f615..36c12a1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -65,12 +65,11 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 3: Build Cube
RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
- final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount);
// base cuboid step
- result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
+ result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
// n dim cuboid steps
for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
- result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i));
+ result.addTask(createNDimensionCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, i - 1), getCuboidOutputPathsByLevel(cuboidRootPath, i), i));
}
outputSide.addStepPhase3_BuildCube(result, cuboidRootPath);
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 700f821..dd866bd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -76,14 +76,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return result;
}
- private void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
+ protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
final int maxLevel = seg.getCubeDesc().getBuildLevel();
- final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, maxLevel);
// base cuboid step
- result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
+ result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
// n dim cuboid steps
for (int i = 1; i <= maxLevel; i++) {
- result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i, jobId));
+ result.addTask(createNDimensionCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, i-1), getCuboidOutputPathsByLevel(cuboidRootPath, i-1), i, jobId));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 14252ee..696b22a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -193,23 +193,11 @@ public class JobBuilderSupport {
return buf.append(" -").append(paraName).append(" ").append(paraValue);
}
- public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
- String[] paths = new String[levels + 1];
- for (int i = 0; i <= levels; i++) {
- if (i == 0) {
- paths[i] = cuboidRootPath + "base_cuboid";
- } else {
- paths[i] = cuboidRootPath + "level_" + i + "_cuboid";
- }
- }
- return paths;
- }
-
public static String getCuboidOutputPathsByLevel(String cuboidRootPath, int level) {
if (level == 0) {
return cuboidRootPath + "base_cuboid";
} else {
- return cuboidRootPath + level + "level_cuboid";
+ return cuboidRootPath + "level_" + level + "_cuboid";
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 9431468..55e11c4 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -41,6 +41,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
super(newSegment, submitter);
}
+ @Override
protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 53c1f96..93cce81 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -260,7 +260,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
// aggregate to ND cuboids
PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
- for (level = 1; level < totalLevels; level++) {
+ for (level = 1; level <= totalLevels; level++) {
partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
logger.info("Level " + level + " partition number: " + partition);
allRDDs[level] = allRDDs[level - 1].flatMapToPair(flatMapFunction).reduceByKey(reducerFunction2, partition).persist(storageLevel);
http://git-wip-us.apache.org/repos/asf/kylin/blob/66bca9a6/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
index edd9460..76e4521 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
@@ -50,7 +50,6 @@ public class SparkCubingJobBuilder extends JobBuilderSupport {
public DefaultChainedExecutable build() {
final CubingJob result = CubingJob.createBuildJob(seg, submitter, config);
- final String jobId = result.getId();
inputSide.addStepPhase1_CreateFlatTable(result);
final IJoinedFlatTableDesc joinedFlatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg);
[05/50] [abbrv] kylin git commit: minor, set reducer number in job
Posted by li...@apache.org.
minor, set reducer number in job
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b736175f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b736175f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b736175f
Branch: refs/heads/master-hbase1.x
Commit: b736175f417f48fe07d01c0e2c99e040c3242d6f
Parents: df5faf3
Author: Roger Shi <ro...@Rogers-MacBook-Pro.local>
Authored: Thu Jan 5 15:38:51 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 5 15:51:01 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 2 +-
.../apache/kylin/engine/mr/steps/LayerReducerNumSizing.java | 8 ++------
.../org/apache/kylin/engine/mr/steps/MergeCuboidJob.java | 2 +-
3 files changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index ef25b55..b2e186d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -131,7 +131,7 @@ public class CuboidJob extends AbstractHadoopJob {
// add metadata to distributed cache
attachSegmentMetadataWithDict(segment, job.getConfiguration());
- LayerReducerNumSizing.setReduceTaskNum(job, segment, getTotalMapInputMB(), nCuboidLevel);
+ job.setNumReduceTasks(LayerReducerNumSizing.getReduceTaskNum(segment, getTotalMapInputMB(), nCuboidLevel));
this.deletePath(job.getConfiguration(), output);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
index 713a95c..7ce9842 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReducerNumSizing.java
@@ -20,8 +20,6 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Reducer;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
@@ -34,7 +32,7 @@ public class LayerReducerNumSizing {
private static final Logger logger = LoggerFactory.getLogger(LayerReducerNumSizing.class);
- public static void setReduceTaskNum(Job job, CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
+ public static int getReduceTaskNum(CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
CubeDesc cubeDesc = cubeSegment.getCubeDesc();
KylinConfig kylinConfig = cubeDesc.getConfig();
@@ -76,9 +74,7 @@ public class LayerReducerNumSizing {
// no more than 500 reducer by default
numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
- job.setNumReduceTasks(numReduceTasks);
-
- logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
+ return numReduceTasks;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b736175f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
index d9ff616..84b76e3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
@@ -82,7 +82,7 @@ public class MergeCuboidJob extends CuboidJob {
// TODO actually only dictionaries from merging segments are needed
attachCubeMetadataWithDict(cube, job.getConfiguration());
- LayerReducerNumSizing.setReduceTaskNum(job, cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
+ job.setNumReduceTasks(LayerReducerNumSizing.getReduceTaskNum(cube.getSegmentById(segmentID), getTotalMapInputMB(), -1));
this.deletePath(job.getConfiguration(), output);
[32/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
new file mode 100644
index 0000000..07b636b
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BaseCuboidBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ */
+public class BaseCuboidBuilder implements java.io.Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(BaseCuboidBuilder.class);
+ public static final String HIVE_NULL = "\\N";
+ protected String cubeName;
+ protected Cuboid baseCuboid;
+ protected CubeDesc cubeDesc;
+ protected CubeSegment cubeSegment;
+ protected Set<String> nullStrs;
+ protected CubeJoinedFlatTableEnrich intermediateTableDesc;
+ protected MeasureIngester<?>[] aggrIngesters;
+ protected Map<TblColRef, Dictionary<String>> dictionaryMap;
+ protected AbstractRowKeyEncoder rowKeyEncoder;
+ protected BufferedMeasureCodec measureCodec;
+
+ protected KylinConfig kylinConfig;
+
+ public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc,
+ AbstractRowKeyEncoder rowKeyEncoder, MeasureIngester<?>[] aggrIngesters, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ this.kylinConfig = kylinConfig;
+ this.cubeDesc = cubeDesc;
+ this.cubeSegment = cubeSegment;
+ this.intermediateTableDesc = intermediateTableDesc;
+ this.rowKeyEncoder = rowKeyEncoder;
+ this.aggrIngesters = aggrIngesters;
+ this.dictionaryMap = dictionaryMap;
+
+ init();
+ measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ }
+
+ public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc) {
+ this.kylinConfig = kylinConfig;
+ this.cubeDesc = cubeDesc;
+ this.cubeSegment = cubeSegment;
+ this.intermediateTableDesc = intermediateTableDesc;
+
+ init();
+ rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
+ measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
+ dictionaryMap = cubeSegment.buildDictionaryMap();
+
+ }
+
+ private void init() {
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ initNullBytes();
+ }
+
+ private void initNullBytes() {
+ nullStrs = Sets.newHashSet();
+ nullStrs.add(HIVE_NULL);
+ String[] nullStrings = cubeDesc.getNullStrings();
+ if (nullStrings != null) {
+ for (String s : nullStrings) {
+ nullStrs.add(s);
+ }
+ }
+ }
+
+ protected boolean isNull(String v) {
+ return nullStrs.contains(v);
+ }
+
+ public byte[] buildKey(String[] flatRow) {
+ int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
+ List<TblColRef> columns = baseCuboid.getColumns();
+ String[] colValues = new String[columns.size()];
+ for (int i = 0; i < columns.size(); i++) {
+ colValues[i] = getCell(rowKeyColumnIndexes[i], flatRow);
+ }
+ return rowKeyEncoder.encode(colValues);
+ }
+
+ public ByteBuffer buildValue(String[] flatRow) {
+ return measureCodec.encode(buildValueObjects(flatRow));
+ }
+
+ public Object[] buildValueObjects(String[] flatRow) {
+ Object[] measures = new Object[cubeDesc.getMeasures().size()];
+ for (int i = 0; i < measures.length; i++) {
+ measures[i] = buildValueOf(i, flatRow);
+ }
+
+ return measures;
+ }
+
+ public void resetAggrs() {
+ for (int i = 0; i < cubeDesc.getMeasures().size(); i++) {
+ aggrIngesters[i].reset();
+ }
+ }
+
+ private Object buildValueOf(int idxOfMeasure, String[] flatRow) {
+ MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
+ FunctionDesc function = measure.getFunction();
+ int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
+
+ int paramCount = function.getParameterCount();
+ String[] inputToMeasure = new String[paramCount];
+
+ // pick up parameter values
+ ParameterDesc param = function.getParameter();
+ int colParamIdx = 0; // index among parameters of column type
+ for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
+ String value;
+ if (function.isCount()) {
+ value = "1";
+ } else if (param.isColumnType()) {
+ value = getCell(colIdxOnFlatTable[colParamIdx++], flatRow);
+ } else {
+ value = param.getValue();
+ }
+ inputToMeasure[i] = value;
+ }
+
+ return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
+ }
+
+ private String getCell(int i, String[] flatRow) {
+ if (isNull(flatRow[i]))
+ return null;
+ else
+ return flatRow[i];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index ffba181..4011915 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -262,6 +262,11 @@ public class CubeStatsReader {
return ret;
}
+ public List<Long> getCuboidsByLayer(int level) {
+ List<List<Long>> layeredCuboids = cuboidScheduler.getCuboidsByLayer();
+ return layeredCuboids.get(level);
+ }
+
private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
int dimensionCount = Long.bitCount(baseCuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
new file mode 100644
index 0000000..4e98618
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/NDCuboidBuilder.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.kv.RowKeyEncoder;
+import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+
+/**
+ */
+public class NDCuboidBuilder implements Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(NDCuboidBuilder.class);
+ protected String cubeName;
+ protected String segmentID;
+ protected CubeSegment cubeSegment;
+ private RowKeySplitter rowKeySplitter;
+ private RowKeyEncoderProvider rowKeyEncoderProvider;
+ private byte[] newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];
+ private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
+
+ public NDCuboidBuilder(CubeSegment cubeSegment) {
+ this.cubeSegment = cubeSegment;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ this.rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
+ }
+
+ public NDCuboidBuilder(CubeSegment cubeSegment, RowKeyEncoderProvider rowKeyEncoderProvider) {
+ this.cubeSegment = cubeSegment;
+ this.rowKeyEncoderProvider = rowKeyEncoderProvider;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ }
+
+
+ public Pair<Integer, ByteArray> buildKey(Cuboid parentCuboid, Cuboid childCuboid, SplittedBytes[] splitBuffers) {
+ RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(childCuboid);
+
+ int offset = 0;
+
+ // rowkey columns
+ long mask = Long.highestOneBit(parentCuboid.getId());
+ long parentCuboidId = parentCuboid.getId();
+ long childCuboidId = childCuboid.getId();
+ long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
+ int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
+ for (int i = 0; i < parentCuboidIdActualLength; i++) {
+ if ((mask & parentCuboidId) > 0) {// if the this bit position equals
+ // 1
+ if ((mask & childCuboidId) > 0) {// if the child cuboid has this
+ // column
+ System.arraycopy(splitBuffers[index].value, 0, newKeyBodyBuf, offset, splitBuffers[index].length);
+ offset += splitBuffers[index].length;
+ }
+ index++;
+ }
+ mask = mask >> 1;
+ }
+
+ int fullKeySize = rowkeyEncoder.getBytesLength();
+ while (newKeyBuf.array().length < fullKeySize) {
+ newKeyBuf.set(new byte[newKeyBuf.length() * 2]);
+ }
+ newKeyBuf.set(0, fullKeySize);
+
+ rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf);
+
+ return new Pair<>(Integer.valueOf(fullKeySize), newKeyBuf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 7b719e0..d08e29a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -18,38 +18,25 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.engine.mr.common.BaseCuboidBuilder;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
/**
*/
@@ -59,131 +46,37 @@ abstract public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
public static final byte[] ONE = Bytes.toBytes("1");
protected String cubeName;
protected String segmentID;
- protected Cuboid baseCuboid;
protected CubeInstance cube;
protected CubeDesc cubeDesc;
protected CubeSegment cubeSegment;
- protected Set<String> nullStrs;
- protected CubeJoinedFlatTableEnrich intermediateTableDesc;
- protected String intermediateTableRowDelimiter;
- protected byte byteRowDelimiter;
protected int counter;
- protected MeasureIngester<?>[] aggrIngesters;
- protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected Object[] measures;
- protected AbstractRowKeyEncoder rowKeyEncoder;
- protected BufferedMeasureCodec measureCodec;
private int errorRecordCounter;
protected Text outputKey = new Text();
protected Text outputValue = new Text();
+ private BaseCuboidBuilder baseCuboidBuilder;
+
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
-
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
- intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
- if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
- throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
- }
-
- byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];
-
- KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
-
- cube = CubeManager.getInstance(config).getCube(cubeName);
+ final KylinConfig kylinConfig = AbstractHadoopJob.loadKylinPropsAndMetadata();
+ cube = CubeManager.getInstance(kylinConfig).getCube(cubeName);
cubeDesc = cube.getDescriptor();
cubeSegment = cube.getSegmentById(segmentID);
+ CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+ baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, cubeDesc, cubeSegment, intermediateTableDesc);
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
- intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
-
- rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
-
- measureCodec = new BufferedMeasureCodec(cubeDesc.getMeasures());
- measures = new Object[cubeDesc.getMeasures().size()];
-
- aggrIngesters = MeasureIngester.create(cubeDesc.getMeasures());
- dictionaryMap = cubeSegment.buildDictionaryMap();
-
- initNullBytes();
- }
-
- private void initNullBytes() {
- nullStrs = Sets.newHashSet();
- nullStrs.add(HIVE_NULL);
- String[] nullStrings = cubeDesc.getNullStrings();
- if (nullStrings != null) {
- for (String s : nullStrings) {
- nullStrs.add(s);
- }
- }
- }
-
- protected boolean isNull(String v) {
- return nullStrs.contains(v);
- }
-
- protected byte[] buildKey(String[] flatRow) {
- int[] rowKeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
- List<TblColRef> columns = baseCuboid.getColumns();
- String[] colValues = new String[columns.size()];
- for (int i = 0; i < columns.size(); i++) {
- colValues[i] = getCell(rowKeyColumnIndexes[i], flatRow);
- }
- return rowKeyEncoder.encode(colValues);
}
- private ByteBuffer buildValue(String[] flatRow) {
-
- for (int i = 0; i < measures.length; i++) {
- measures[i] = buildValueOf(i, flatRow);
- }
-
- return measureCodec.encode(measures);
- }
-
- private Object buildValueOf(int idxOfMeasure, String[] flatRow) {
- MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
- FunctionDesc function = measure.getFunction();
- int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
-
- int paramCount = function.getParameterCount();
- String[] inputToMeasure = new String[paramCount];
-
- // pick up parameter values
- ParameterDesc param = function.getParameter();
- int colParamIdx = 0; // index among parameters of column type
- for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
- String value;
- if (function.isCount()) {
- value = "1";
- } else if (param.isColumnType()) {
- value = getCell(colIdxOnFlatTable[colParamIdx++], flatRow);
- } else {
- value = param.getValue();
- }
- inputToMeasure[i] = value;
- }
-
- return aggrIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
- }
-
- private String getCell(int i, String[] flatRow) {
- if (isNull(flatRow[i]))
- return null;
- else
- return flatRow[i];
- }
protected void outputKV(String[] flatRow, Context context) throws IOException, InterruptedException {
- byte[] rowKey = buildKey(flatRow);
+ byte[] rowKey = baseCuboidBuilder.buildKey(flatRow);
outputKey.set(rowKey, 0, rowKey.length);
- ByteBuffer valueBuf = buildValue(flatRow);
+ ByteBuffer valueBuf = baseCuboidBuilder.buildValue(flatRow);
outputValue.set(valueBuf.array(), 0, valueBuf.position());
context.write(outputKey, outputValue);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 01cdd4a..b924edc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -18,29 +18,27 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.util.Collection;
-
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.SplittedBytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.RowKeySplitter;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyEncoder;
-import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Collection;
+
/**
* @author George Song (ysong1)
*
@@ -59,10 +57,9 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
private int handleCounter;
private int skipCounter;
- private byte[] newKeyBodyBuf = new byte[RowConstants.ROWKEY_BUFFER_SIZE];
- private ByteArray newKeyBuf = ByteArray.allocate(RowConstants.ROWKEY_BUFFER_SIZE);
private RowKeySplitter rowKeySplitter;
- private RowKeyEncoderProvider rowKeyEncoderProvider;
+
+ private NDCuboidBuilder ndCuboidBuilder;
@Override
protected void setup(Context context) throws IOException {
@@ -76,48 +73,13 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
cubeSegment = cube.getSegmentById(segmentID);
cubeDesc = cube.getDescriptor();
-
+ ndCuboidBuilder = new NDCuboidBuilder(cubeSegment);
// initialize CubiodScheduler
cuboidScheduler = new CuboidScheduler(cubeDesc);
-
rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
- rowKeyEncoderProvider = new RowKeyEncoderProvider(cubeSegment);
}
- private int buildKey(Cuboid parentCuboid, Cuboid childCuboid, SplittedBytes[] splitBuffers) {
- RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(childCuboid);
-
- int offset = 0;
-
- // rowkey columns
- long mask = Long.highestOneBit(parentCuboid.getId());
- long parentCuboidId = parentCuboid.getId();
- long childCuboidId = childCuboid.getId();
- long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(parentCuboid.getId());
- int index = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId
- for (int i = 0; i < parentCuboidIdActualLength; i++) {
- if ((mask & parentCuboidId) > 0) {// if the this bit position equals
- // 1
- if ((mask & childCuboidId) > 0) {// if the child cuboid has this
- // column
- System.arraycopy(splitBuffers[index].value, 0, newKeyBodyBuf, offset, splitBuffers[index].length);
- offset += splitBuffers[index].length;
- }
- index++;
- }
- mask = mask >> 1;
- }
- int fullKeySize = rowkeyEncoder.getBytesLength();
- while (newKeyBuf.array().length < fullKeySize) {
- newKeyBuf.set(new byte[newKeyBuf.length() * 2]);
- }
- newKeyBuf.set(0, fullKeySize);
-
- rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf);
-
- return fullKeySize;
- }
@Override
public void doMap(Text key, Text value, Context context) throws IOException, InterruptedException {
@@ -143,8 +105,8 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
for (Long child : myChildren) {
Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
- int fullKeySize = buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
- outputKey.set(newKeyBuf.array(), 0, fullKeySize);
+ Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
+ outputKey.set(result.getSecond().array(), 0, result.getFirst());
context.write(outputKey, value);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
index 97dd750..29ca9b8 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java
@@ -41,7 +41,6 @@ import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -161,7 +160,7 @@ public class CubeReducerTest extends LocalFileMetadataTestCase {
}
private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) {
- Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count), new LongMutable(item_count) };
+ Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) };
ByteBuffer buf = codec.encode(values);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 4921dca..be2cdf6 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -43,6 +43,11 @@
<artifactId>kylin-core-job</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-mr</artifactId>
+ </dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
@@ -111,6 +116,10 @@
<artifactId>maven-model</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
new file mode 100644
index 0000000..a7a4151
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine2.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.MRBatchCubingEngine2;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+
+/**
+ */
+public class SparkBatchCubingEngine2 extends MRBatchCubingEngine2 {
+ @Override
+ public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
+ return new SparkBatchCubingJobBuilder2(newSegment, submitter).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
new file mode 100644
index 0000000..9532d31
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark;
+
+import org.apache.hadoop.util.ClassUtil;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.BatchCubingJobBuilder2;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ */
+public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
+
+ private static final Logger logger = LoggerFactory.getLogger(SparkBatchCubingJobBuilder2.class);
+
+ public SparkBatchCubingJobBuilder2(CubeSegment newSegment, String submitter) {
+ super(newSegment, submitter);
+ }
+
+ protected void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
+
+ }
+
+ @Override
+ protected AbstractExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
+ IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg);
+ final SparkExecutable sparkExecutable = new SparkExecutable();
+ sparkExecutable.setClassName(SparkCubingByLayer.class.getName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_PATH.getOpt(), flatTableDesc.getTableName());
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), "/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/"); //FIXME
+ sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath);
+
+ StringBuilder jars = new StringBuilder();
+
+ StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration"));
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
+
+ StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
+ sparkExecutable.setJars(jars.toString());
+ // sparkExecutable.setJars("/Users/shishaofeng/.m2/repository/org/cloudera/htrace/htrace-core/2.01/htrace-core-2.01.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-protocol/0.98.8-hadoop2/hbase-protocol-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2/repository/org/apache/hbase/hbase-common/0.98.8-hadoop2/hbase-common-0.98.8-hadoop2.jar,/Users/shishaofeng/.m2//repository/org/apache/hbase/hbase-client/0.98.8-hadoop2/hbase-client-0.98.8-hadoop2.jar");
+
+ sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE + " with Spark");
+ return sparkExecutable;
+
+ }
+
+ private String findJar(String className) {
+ try {
+ return ClassUtil.findContainingJar(Class.forName(className));
+ } catch (ClassNotFoundException e) {
+ logger.error("failed to locate jar for class " + className, e);
+ }
+
+ return "";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 4ec5048..0437a80 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -84,6 +84,7 @@ import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregators;
import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -153,6 +154,20 @@ public class SparkCubing extends AbstractApplication {
return options;
}
+ public static KylinConfig loadKylinPropsAndMetadata(String folder) throws IOException {
+ File metaDir = new File(folder);
+ if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) {
+ System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath());
+ logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath());
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ System.out.println("setting metadataUrl to " + metaDir.getAbsolutePath());
+ kylinConfig.setMetadataUrl(metaDir.getAbsolutePath());
+ return kylinConfig;
+ } else {
+ return KylinConfig.getInstanceFromEnv();
+ }
+ }
+
private void setupClasspath(JavaSparkContext sc, String confPath) throws Exception {
ClassUtil.addClasspath(confPath);
final File[] files = new File(confPath).listFiles(new FileFilter() {
@@ -466,7 +481,7 @@ public class SparkCubing extends AbstractApplication {
}).saveAsNewAPIHadoopFile(hFileLocation, ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class, conf);
}
- private static void prepare() throws Exception {
+ public static void prepare() throws Exception {
final File file = new File(SparkFiles.get("kylin.properties"));
final String confPath = file.getParentFile().getAbsolutePath();
System.out.println("conf directory:" + confPath);
@@ -530,12 +545,18 @@ public class SparkCubing extends AbstractApplication {
}
}
- private Collection<String> getKyroClasses() {
+ public static Collection<String> getKyroClasses() {
Set<Class> kyroClasses = Sets.newHashSet();
kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class));
+ kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class));
+
kyroClasses.add(HashMap.class);
kyroClasses.add(org.apache.spark.sql.Row[].class);
kyroClasses.add(org.apache.spark.sql.Row.class);
@@ -545,11 +566,15 @@ public class SparkCubing extends AbstractApplication {
kyroClasses.add(org.apache.spark.sql.types.StructField.class);
kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
- kyroClasses.add(Object[].class);
kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
kyroClasses.add(Hashing.murmur3_128().getClass());
- kyroClasses.add(org.apache.spark.sql.columnar.CachedBatch.class);
+ kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
+ kyroClasses.add(Object[].class);
+ kyroClasses.add(int[].class);
+ kyroClasses.add(byte[].class);
kyroClasses.add(byte[][].class);
+ kyroClasses.add(String[].class);
+ kyroClasses.add(String[][].class);
kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
kyroClasses.add(scala.math.BigDecimal.class);
kyroClasses.add(java.math.BigDecimal.class);
@@ -557,6 +582,61 @@ public class SparkCubing extends AbstractApplication {
kyroClasses.add(java.math.RoundingMode.class);
kyroClasses.add(java.util.ArrayList.class);
kyroClasses.add(java.util.LinkedList.class);
+ kyroClasses.add(java.util.HashSet.class);
+ kyroClasses.add(java.util.LinkedHashSet.class);
+ kyroClasses.add(java.util.LinkedHashMap.class);
+ kyroClasses.add(java.util.TreeMap.class);
+ kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
+
+ kyroClasses.add(java.util.HashMap.class);
+ kyroClasses.add(java.util.Properties.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+ kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class);
+ kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class);
+ kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
+ kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class);
+ kyroClasses.add(org.apache.kylin.common.util.Array.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
+ kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
+ kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+ kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
+ kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
+ kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
+ kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+ kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+ kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+ kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
+ kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+ kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+ kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
+
+ try {
+ kyroClasses.add(Class.forName("com.google.common.collect.EmptyImmutableList"));
+ } catch (ClassNotFoundException e) {
+ logger.error("failed to load class", e);
+ }
ArrayList<String> result = Lists.newArrayList();
for (Class kyroClass : kyroClasses) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
new file mode 100644
index 0000000..53c1f96
--- /dev/null
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.engine.spark;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.common.RowKeySplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.cuboid.CuboidScheduler;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.kv.RowKeyEncoderProvider;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.engine.EngineFactory;
+import org.apache.kylin.engine.mr.BatchCubingJobBuilder2;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.engine.mr.common.BaseCuboidBuilder;
+import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.engine.mr.common.NDCuboidBuilder;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkFiles;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.hive.HiveContext;
+import org.apache.spark.storage.StorageLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.kylin.engine.spark.SparkCubing.getKyroClasses;
+
+/**
+ */
+public class SparkCubingByLayer extends AbstractApplication implements Serializable {
+
+ protected static final Logger logger = LoggerFactory.getLogger(SparkCubingByLayer.class);
+
+ public static final Option OPTION_INPUT_PATH = OptionBuilder.withArgName("path").hasArg().isRequired(true).withDescription("Hive Intermediate Table").create("hiveTable");
+ public static final Option OPTION_CUBE_NAME = OptionBuilder.withArgName(BatchConstants.ARG_CUBE_NAME).hasArg().isRequired(true).withDescription("Cube Name").create(BatchConstants.ARG_CUBE_NAME);
+ public static final Option OPTION_SEGMENT_ID = OptionBuilder.withArgName("segment").hasArg().isRequired(true).withDescription("Cube Segment Id").create("segmentId");
+ public static final Option OPTION_CONF_PATH = OptionBuilder.withArgName("confPath").hasArg().isRequired(true).withDescription("Configuration Path").create("confPath");
+ public static final Option OPTION_OUTPUT_PATH = OptionBuilder.withArgName(BatchConstants.ARG_OUTPUT).hasArg().isRequired(true).withDescription("Cube output path").create(BatchConstants.ARG_OUTPUT);
+
+ private Options options;
+
+ public SparkCubingByLayer() {
+ options = new Options();
+ options.addOption(OPTION_INPUT_PATH);
+ options.addOption(OPTION_CUBE_NAME);
+ options.addOption(OPTION_SEGMENT_ID);
+ options.addOption(OPTION_CONF_PATH);
+ options.addOption(OPTION_OUTPUT_PATH);
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ private void setupClasspath(JavaSparkContext sc, String confPath) throws Exception {
+ ClassUtil.addClasspath(confPath);
+ final File[] files = new File(confPath).listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ if (pathname.getAbsolutePath().endsWith(".xml")) {
+ return true;
+ }
+ if (pathname.getAbsolutePath().endsWith(".properties")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ for (File file : files) {
+ sc.addFile(file.getAbsolutePath());
+ }
+ }
+
+ private static final void prepare() {
+ final File file = new File(SparkFiles.get("kylin.properties"));
+ final String confPath = file.getParentFile().getAbsolutePath();
+ logger.info("conf directory:" + confPath);
+ System.setProperty(KylinConfig.KYLIN_CONF, confPath);
+ ClassUtil.addClasspath(confPath);
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ final String hiveTable = optionsHelper.getOptionValue(OPTION_INPUT_PATH);
+ final String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
+ final String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
+ final String confPath = optionsHelper.getOptionValue(OPTION_CONF_PATH);
+ final String outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT_PATH);
+
+ SparkConf conf = new SparkConf().setAppName("Cubing Application");
+ //serialization conf
+ conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ conf.set("spark.kryo.registrationRequired", "true");
+ final Iterable<String> allClasses = Iterables.filter(Iterables.concat(Lists.newArrayList(conf.get("spark.kryo.classesToRegister", "").split(",")), getKyroClasses()), new Predicate<String>() {
+ @Override
+ public boolean apply(@Nullable String input) {
+ return input != null && input.trim().length() > 0;
+ }
+ });
+ conf.set("spark.kryo.classesToRegister", StringUtils.join(allClasses, ","));
+
+ JavaSparkContext sc = new JavaSparkContext(conf);
+ setupClasspath(sc, confPath);
+ HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
+
+ HiveContext sqlContext = new HiveContext(sc.sc());
+ final DataFrame intermediateTable = sqlContext.table(hiveTable);
+
+ System.setProperty(KylinConfig.KYLIN_CONF, confPath);
+ final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
+ final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
+ final CubeDesc cubeDesc = cubeInstance.getDescriptor();
+ final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
+ final CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
+
+ final KylinConfig kylinConfig = cubeDesc.getConfig();
+ final Broadcast<CubeDesc> vCubeDesc = sc.broadcast(cubeDesc);
+ final Broadcast<CubeSegment> vCubeSegment = sc.broadcast(cubeSegment);
+ final NDCuboidBuilder ndCuboidBuilder = new NDCuboidBuilder(vCubeSegment.getValue(), new RowKeyEncoderProvider(vCubeSegment.getValue()));
+
+ final Broadcast<CuboidScheduler> vCuboidScheduler = sc.broadcast(new CuboidScheduler(vCubeDesc.getValue()));
+
+ final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ final Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+ final int measureNum = cubeDesc.getMeasures().size();
+ final BaseCuboidBuilder baseCuboidBuilder = new BaseCuboidBuilder(kylinConfig, vCubeDesc.getValue(), vCubeSegment.getValue(), intermediateTableDesc, AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid), MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
+
+ int countMeasureIndex = 0;
+ for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
+ if (measureDesc.getFunction().isCount() == true) {
+ break;
+ } else {
+ countMeasureIndex++;
+ }
+ }
+ final CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
+ boolean[] needAggr = new boolean[cubeDesc.getMeasures().size()];
+ boolean allNormalMeasure = true;
+ for (int i = 0; i < cubeDesc.getMeasures().size(); i++) {
+ needAggr[i] = !cubeDesc.getMeasures().get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
+ allNormalMeasure = allNormalMeasure && needAggr[i];
+ }
+ logger.info("All measure are normal (agg on all cuboids) ? : " + allNormalMeasure);
+
+ StorageLevel storageLevel = StorageLevel.MEMORY_AND_DISK_SER();
+
+ // encode with dimension encoding, transform to <ByteArray, Object[]> RDD
+ final JavaPairRDD<ByteArray, Object[]> encodedBaseRDD = intermediateTable.javaRDD().mapToPair(new PairFunction<Row, ByteArray, Object[]>() {
+ transient boolean initialized = false;
+
+ @Override
+ public Tuple2<ByteArray, Object[]> call(Row row) throws Exception {
+ if (initialized == false) {
+ prepare();
+ initialized = true;
+ }
+
+ String[] rowArray = rowToArray(row);
+ baseCuboidBuilder.resetAggrs();
+ byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
+ Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
+ return new Tuple2<>(new ByteArray(rowKey), result);
+ }
+
+ private String[] rowToArray(Row row) {
+ String[] result = new String[row.size()];
+ for (int i = 0; i < row.size(); i++) {
+ final Object o = row.get(i);
+ if (o != null) {
+ result[i] = o.toString();
+ } else {
+ result[i] = null;
+ }
+ }
+ return result;
+ }
+
+ });
+
+ logger.info("encodedBaseRDD partition number: " + encodedBaseRDD.getNumPartitions());
+ Long totalCount = 0L;
+ if (kylinConfig.isSparkSanityCheckEnabled()) {
+ totalCount = encodedBaseRDD.count();
+ logger.info("encodedBaseRDD row count: " + encodedBaseRDD.count());
+ }
+
+ final MeasureAggregators measureAggregators = new MeasureAggregators(cubeDesc.getMeasures());
+ final BaseCuboidReducerFunction2 baseCuboidReducerFunction = new BaseCuboidReducerFunction2(measureNum, vCubeDesc.getValue(), measureAggregators);
+ BaseCuboidReducerFunction2 reducerFunction2 = baseCuboidReducerFunction;
+ if (allNormalMeasure == false) {
+ reducerFunction2 = new CuboidReducerFunction2(measureNum, vCubeDesc.getValue(), measureAggregators, needAggr);
+ }
+
+ final int totalLevels = cubeDesc.getBuildLevel();
+ JavaPairRDD<ByteArray, Object[]>[] allRDDs = new JavaPairRDD[totalLevels];
+ int level = 0;
+ int partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
+
+ // aggregate to calculate base cuboid
+ allRDDs[0] = encodedBaseRDD.reduceByKey(baseCuboidReducerFunction, partition).persist(storageLevel);
+ saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, sc.hadoopConfiguration());
+
+ // aggregate to ND cuboids
+ PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
+
+ for (level = 1; level < totalLevels; level++) {
+ partition = estimateRDDPartitionNum(level, cubeStatsReader, kylinConfig);
+ logger.info("Level " + level + " partition number: " + partition);
+ allRDDs[level] = allRDDs[level - 1].flatMapToPair(flatMapFunction).reduceByKey(reducerFunction2, partition).persist(storageLevel);
+ if (kylinConfig.isSparkSanityCheckEnabled() == true) {
+ sanityCheck(allRDDs[level], totalCount, level, cubeStatsReader, countMeasureIndex);
+ }
+ saveToHDFS(allRDDs[level], vCubeDesc.getValue(), outputPath, level, sc.hadoopConfiguration());
+ allRDDs[level - 1].unpersist();
+ }
+ allRDDs[totalLevels - 1].unpersist();
+ logger.info("Finished on calculating all level cuboids.");
+ }
+
+ private static int estimateRDDPartitionNum(int level, CubeStatsReader statsReader, KylinConfig kylinConfig) {
+ double baseCuboidSize = statsReader.estimateLayerSize(level);
+ float rddCut = kylinConfig.getSparkRDDPartitionCutMB();
+ int partition = (int) (baseCuboidSize / rddCut);
+ partition = Math.max(kylinConfig.getSparkMinPartition(), partition);
+ partition = Math.min(kylinConfig.getSparkMaxPartition(), partition);
+ logger.debug("Estimated level " + level + " partition number: " + partition);
+ return partition;
+ }
+
+ private static void saveToHDFS(final JavaPairRDD<ByteArray, Object[]> rdd, final CubeDesc cubeDesc, final String hdfsBaseLocation, int level, Configuration conf) {
+ final String cuboidOutputPath = BatchCubingJobBuilder2.getCuboidOutputPathsByLevel(hdfsBaseLocation, level);
+ rdd.mapToPair(new PairFunction<Tuple2<ByteArray, Object[]>, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text>() {
+ BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());
+ @Override
+ public Tuple2<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ ByteBuffer valueBuf = codec.encode(tuple2._2());
+ byte[] encodedBytes = new byte[valueBuf.position()];
+ System.arraycopy(valueBuf.array(), 0, encodedBytes, 0, valueBuf.position());
+ return new Tuple2<>(new org.apache.hadoop.io.Text(tuple2._1().array()), new org.apache.hadoop.io.Text(encodedBytes));
+ }
+ }).saveAsNewAPIHadoopFile(cuboidOutputPath, org.apache.hadoop.io.Text.class, org.apache.hadoop.io.Text.class, SequenceFileOutputFormat.class, conf);
+ logger.info("Persisting RDD for level " + level + " into " + cuboidOutputPath);
+ }
+
+ class BaseCuboidReducerFunction2 implements Function2<Object[], Object[], Object[]> {
+ CubeDesc cubeDesc;
+ int measureNum;
+ MeasureAggregators aggregators;
+
+ BaseCuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, MeasureAggregators aggregators) {
+ this.cubeDesc = cubeDesc;
+ this.measureNum = measureNum;
+ this.aggregators = aggregators;
+ }
+
+ @Override
+ public Object[] call(Object[] input1, Object[] input2) throws Exception {
+ Object[] result = new Object[measureNum];
+ aggregators.aggregate(input1, input2, result);
+ return result;
+ }
+ }
+
+ class CuboidReducerFunction2 extends BaseCuboidReducerFunction2 {
+ boolean[] needAggr;
+
+ CuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, MeasureAggregators aggregators, boolean[] needAggr) {
+ super(measureNum, cubeDesc, aggregators);
+ this.needAggr = needAggr;
+ }
+
+ @Override
+ public Object[] call(Object[] input1, Object[] input2) throws Exception {
+ Object[] result = new Object[measureNum];
+ aggregators.aggregate(input1, input2, result, needAggr);
+ return result;
+ }
+ }
+
+ private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0);
+
+ class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> {
+
+ CubeSegment cubeSegment;
+ CubeDesc cubeDesc;
+ CuboidScheduler cuboidScheduler;
+ NDCuboidBuilder ndCuboidBuilder;
+ RowKeySplitter rowKeySplitter;
+ transient boolean initialized = false;
+
+ CuboidFlatMap(CubeSegment cubeSegment, CubeDesc cubeDesc, CuboidScheduler cuboidScheduler, NDCuboidBuilder ndCuboidBuilder) {
+ this.cubeSegment = cubeSegment;
+ this.cubeDesc = cubeDesc;
+ this.cuboidScheduler = cuboidScheduler;
+ this.ndCuboidBuilder = ndCuboidBuilder;
+ this.rowKeySplitter = new RowKeySplitter(cubeSegment, 65, 256);
+ }
+
+ @Override
+ public Iterable<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ if (initialized == false) {
+ prepare();
+ initialized = true;
+ }
+
+ byte[] key = tuple2._1().array();
+ long cuboidId = rowKeySplitter.split(key);
+ Cuboid parentCuboid = Cuboid.findById(cubeDesc, cuboidId);
+
+ Collection<Long> myChildren = cuboidScheduler.getSpanningCuboid(cuboidId);
+
+ // if still empty or null
+ if (myChildren == null || myChildren.size() == 0) {
+ return EMTPY_ITERATOR;
+ }
+
+ List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size());
+ for (Long child : myChildren) {
+ Cuboid childCuboid = Cuboid.findById(cubeDesc, child);
+ Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers());
+
+ byte[] newKey = new byte[result.getFirst()];
+ System.arraycopy(result.getSecond().array(), 0, newKey, 0, result.getFirst());
+
+ tuples.add(new Tuple2<>(new ByteArray(newKey), tuple2._2()));
+ }
+
+ return tuples;
+ }
+ }
+
+ //sanity check
+
+ private void sanityCheck(JavaPairRDD<ByteArray, Object[]> rdd, Long totalCount, int thisLevel, CubeStatsReader cubeStatsReader, final int countMeasureIndex) {
+ int thisCuboidNum = cubeStatsReader.getCuboidsByLayer(thisLevel).size();
+ Long count2 = getRDDCountSum(rdd, countMeasureIndex);
+ if (count2 != totalCount * thisCuboidNum) {
+ throw new IllegalStateException(String.format("Sanity check failed, level %s, total count(*) is %s; cuboid number %s", thisLevel, count2, thisCuboidNum));
+ } else {
+ logger.info("sanity check success for level " + thisLevel + ", count(*) is " + (count2 / thisCuboidNum));
+ }
+ }
+
+ private Long getRDDCountSum(JavaPairRDD<ByteArray, Object[]> rdd, final int countMeasureIndex) {
+ final ByteArray ONE = new ByteArray();
+ Long count = rdd.mapValues(new Function<Object[], Long>() {
+ @Override
+ public Long call(Object[] objects) throws Exception {
+ return (Long) objects[countMeasureIndex]; // assume the first measure is COUNT(*)
+ }
+ }).reduce(new Function2<Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>, Tuple2<ByteArray, Long>>() {
+ @Override
+ public Tuple2<ByteArray, Long> call(Tuple2<ByteArray, Long> longTuple2, Tuple2<ByteArray, Long> longTuple22) throws Exception {
+ return new Tuple2<>(ONE, longTuple2._2() + longTuple22._2());
+ }
+ })._2();
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index 7c88372..644f73f 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.Logger;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -38,11 +39,16 @@ public class SparkExecutable extends AbstractExecutable {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(SparkExecutable.class);
private static final String CLASS_NAME = "className";
+ private static final String JARS = "jars";
public void setClassName(String className) {
this.setParam(CLASS_NAME, className);
}
+ public void setJars(String jars) {
+ this.setParam(JARS, jars);
+ }
+
private String formatArgs() {
StringBuilder stringBuilder = new StringBuilder();
for (Map.Entry<String, String> entry : getParams().entrySet()) {
@@ -50,6 +56,9 @@ public class SparkExecutable extends AbstractExecutable {
tmp.append("-").append(entry.getKey()).append(" ").append(entry.getValue()).append(" ");
if (entry.getKey().equals(CLASS_NAME)) {
stringBuilder.insert(0, tmp);
+ } else if (entry.getKey().equals(JARS)) {
+ // JARS is for spark-submit, not for app
+ continue;
} else {
stringBuilder.append(tmp);
}
@@ -65,12 +74,22 @@ public class SparkExecutable extends AbstractExecutable {
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final KylinConfig config = context.getConfig();
Preconditions.checkNotNull(config.getSparkHome());
- Preconditions.checkNotNull(config.getSparkMaster());
+ Preconditions.checkNotNull(config.getKylinJobJarPath());
+ String sparkConf = config.getSparkConfFile();
+ String jars = this.getParam(JARS);
+
+ String jobJar = config.getKylinJobJarPath();
+
+ if (StringUtils.isEmpty(jars)) {
+ jars = jobJar;
+ }
+
try {
- String cmd = String.format("%s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --master %s %s %s", config.getSparkHome(), config.getSparkMaster(), config.getKylinSparkJobJarPath(), formatArgs());
+ String cmd = String.format("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class \"org.apache.kylin.common.util.SparkEntry\" --properties-file %s --jars %s %s %s", config.getSparkHadoopConfDir(), config.getSparkHome(), sparkConf, jars, jobJar, formatArgs());
logger.info("cmd:" + cmd);
final StringBuilder output = new StringBuilder();
- config.getCliCommandExecutor().execute(cmd, new Logger() {
+ CliCommandExecutor exec = new CliCommandExecutor();
+ exec.execute(cmd, new Logger() {
@Override
public void log(String message) {
output.append(message);
@@ -84,4 +103,5 @@ public class SparkExecutable extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
index 4f407ff..793cd87 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/DefaultTupleConverter.java
@@ -38,7 +38,7 @@ import scala.Tuple2;
*/
public final class DefaultTupleConverter implements TupleConverter {
- private final static ThreadLocal<ByteBuffer> valueBuf = new ThreadLocal<>();
+ private final static transient ThreadLocal<ByteBuffer> valueBuf = new ThreadLocal<>();
private final CubeSegment segment;
private final int measureCount;
private final Map<TblColRef, Integer> columnLengthMap;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/examples/test_case_data/sandbox/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin-spark-conf.properties b/examples/test_case_data/sandbox/kylin-spark-conf.properties
new file mode 100644
index 0000000..b4a0c97
--- /dev/null
+++ b/examples/test_case_data/sandbox/kylin-spark-conf.properties
@@ -0,0 +1,28 @@
+spark.yarn.submit.file.replication=1
+spark.yarn.executor.memoryOverhead=200
+spark.yarn.driver.memoryOverhead=384
+#spark.master=local[4]
+#spark.submit.deployMode=client
+spark.master=yarn
+spark.submit.deployMode=cluster
+spark.eventLog.enabled=true
+spark.yarn.scheduler.heartbeat.interval-ms=5000
+spark.yarn.preserve.staging.files=true
+spark.yarn.queue=default
+spark.yarn.containerLauncherMaxThreads=25
+spark.yarn.max.executor.failures=3
+spark.eventLog.dir=hdfs\:///spark-history
+spark.history.kerberos.enabled=true
+spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
+spark.history.ui.port=18080
+spark.history.fs.logDirectory=hdfs\:///spark-history
+spark.executor.memory=1G
+spark.storage.memoryFraction=0.3
+spark.executor.cores=1
+spark.executor.instances=1
+spark.history.kerberos.keytab=none
+spark.history.kerberos.principal=none
+spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+spark.driver.extraJavaOptions=-Dhdp.version=current
+spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 1724619..a011911 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -79,6 +79,8 @@ kylin.job.remote-cli-working-dir=/tmp/kylin
# Max count of concurrent jobs running
kylin.job.max-concurrent-jobs=10
+kylin.source.hive.redistribute-flat-table=false
+
# Time interval to check hadoop job status
kylin.engine.mr.yarn-check-interval-seconds=10
@@ -154,3 +156,11 @@ kylin.server.query-metrics-percentiles-intervals=60, 360, 3600
# Env DEV|QA|PROD
kylin.env=DEV
+kylin.source.hive.keep-flat-table=true
+
+### Spark as Engine ###
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf
+kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox
+kylin.engine.spark.spark-home=/usr/local/spark
+kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties
+kylin.engine.spark.sanity-check-enabled=false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 841f19d..fcb5ce3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,7 +84,7 @@
<commons-math3.version>3.6.1</commons-math3.version>
<!-- Spark -->
- <spark.version>1.3.0</spark.version>
+ <spark.version>1.6.3</spark.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index a9c8321..f905fe1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -327,10 +327,10 @@ public class CubeController extends BasicController {
throw new InternalErrorException("Cannot find cube '" + cubeName + "'");
}
- if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
- int num = cube.getBuildingSegments().size();
- throw new InternalErrorException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment. Discard the related job first."));
- }
+// if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
+// int num = cube.getBuildingSegments().size();
+// throw new InternalErrorException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment. Discard the related job first."));
+// }
return cubeService.purgeCube(cube);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 617cc85..454ed6b 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -253,6 +253,13 @@
<scope>provided</scope>
</dependency>
+ <!-- Spark dependency -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 0f351b0..bd53f9a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -56,8 +57,6 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
-
public class HiveMRInput implements IMRInput {
public static String getTableNameForHCat(TableDesc table) {
@@ -74,7 +73,7 @@ public class HiveMRInput implements IMRInput {
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
return new HiveTableInputFormat(getTableNameForHCat(table));
}
-
+
@Override
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
return new IMRBatchMergeInputSide() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index f0ad6e0..9b1a00d 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -31,7 +31,6 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
@@ -60,8 +59,8 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal sum = new BigDecimal("333.1234567");
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
- LongMutable count = new LongMutable(2);
- LongMutable item_count = new LongMutable(100);
+ Long count = new Long(2);
+ Long item_count = new Long(100);
ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });
buf.flip();
@@ -91,7 +90,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
LongWritable count = new LongWritable(2);
- LongMutable item_count = new LongMutable(100);
+ Long item_count = new Long(100);
codec.encode(new Object[] { sum, min, max, count, item_count });
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index aae0dab..919a903 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -43,6 +43,10 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-spark</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-source-kafka</artifactId>
</dependency>
<dependency>
[04/50] [abbrv] kylin git commit: KYLIN-2333 revert the changes as there is the case that “mandatory only combination is allowed”
Posted by li...@apache.org.
KYLIN-2333 revert the changes as there is the case that \u201cmandatory only combination is allowed\u201d
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df5faf37
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df5faf37
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df5faf37
Branch: refs/heads/master-hbase1.x
Commit: df5faf37cd3186376c28018a5a5a7e42bcea1db9
Parents: 4b5257a
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 15:14:37 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 15:14:37 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java | 3 +--
.../java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java | 2 +-
.../main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java | 2 +-
3 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 8fbecd3..456f615 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -65,12 +65,11 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
// Phase 3: Build Cube
RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
- final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount);
// base cuboid step
result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
// n dim cuboid steps
- for (int i = 1; i < groupRowkeyColumnsCount; i++) {
+ for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i));
}
outputSide.addStepPhase3_BuildCube(result, cuboidRootPath);
http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 9229a75..1e3f078 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -81,7 +81,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
// base cuboid step
result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
// n dim cuboid steps
- for (int i = 1; i < maxLevel; i++) {
+ for (int i = 1; i <= maxLevel; i++) {
result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i, jobId));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/df5faf37/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 23fce2b..122b8ba 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -194,7 +194,7 @@ public class JobBuilderSupport {
public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
String[] paths = new String[levels];
- for (int i = 0; i < levels; i++) {
+ for (int i = 0; i <= levels; i++) {
if (i == 0) {
paths[i] = cuboidRootPath + "base_cuboid";
} else {
[15/50] [abbrv] kylin git commit: KYLIN-2362 Unify shell interpreter
in scripts
Posted by li...@apache.org.
KYLIN-2362 Unify shell interpreter in scripts
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fbacb71e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fbacb71e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fbacb71e
Branch: refs/heads/master-hbase1.x
Commit: fbacb71ed7aa982b04dc4556c253a9bcfee576fa
Parents: 40ede31
Author: etherge <et...@163.com>
Authored: Fri Jan 6 14:18:47 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Jan 6 15:53:52 2017 +0800
----------------------------------------------------------------------
build/bin/check-env.sh | 2 +-
build/bin/find-hive-dependency.sh | 4 ++--
build/bin/kylin.sh | 2 +-
build/bin/sample.sh | 8 ++++----
4 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fbacb71e/build/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index 9cd8a64..a4003c9 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -41,7 +41,7 @@ then
quit "Please make sure the user has the privilege to run hadoop shell"
fi
-WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.env.hdfs-working-dir`
+WORKING_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.env.hdfs-working-dir`
hadoop fs -mkdir -p $WORKING_DIR
if [ $? != 0 ]
then
http://git-wip-us.apache.org/repos/asf/kylin/blob/fbacb71e/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index fe9e0a3..453a35a 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -19,13 +19,13 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
+client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
hive_env=
echo Retrieving hive dependency...
if [ "${client_mode}" == "beeline" ]
then
- beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
+ beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
hive_env=`beeline ${beeline_params} --outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' `
else
hive_env=`hive -e set 2>&1 | grep 'env:CLASSPATH'`
http://git-wip-us.apache.org/repos/asf/kylin/blob/fbacb71e/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index fca1e5c..0cdbbc6 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -73,7 +73,7 @@ then
#In this way we no longer need to explicitly configure hadoop/hbase related classpath for tomcat,
#hbase command will do all the dirty tasks for us:
- spring_profile=`sh ${dir}/get-properties.sh kylin.security.profile`
+ spring_profile=`bash ${dir}/get-properties.sh kylin.security.profile`
if [ -z "$spring_profile" ]
then
quit 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.'
http://git-wip-us.apache.org/repos/asf/kylin/blob/fbacb71e/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 1397034..1363a5d 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -29,10 +29,10 @@ echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data"
hadoop fs -mkdir -p /tmp/kylin/sample_cube/data
hadoop fs -put * /tmp/kylin/sample_cube/data/
-hive_client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
+hive_client_mode=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client`
if [ "${hive_client_mode}" == "beeline" ]
then
- beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
+ beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
@@ -42,8 +42,8 @@ echo "Sample hive tables are created successfully; Going to create sample cube..
hadoop fs -rm -r /tmp/kylin/sample_cube
# set engine type and storage type to cube desc
-default_engine_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default`
-default_storage_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.storage.default`
+default_engine_type=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default`
+default_storage_type=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.storage.default`
if [ -z "$default_engine_type" ]; then
default_engine_type=2
default_storage_type=2
[33/50] [abbrv] kylin git commit: KYLIN-2331 by layer spark cubing
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
index 8951b69..710f324 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregators.java
@@ -84,6 +84,24 @@ public class MeasureAggregators implements Serializable {
}
}
+ public void aggregate(Object[] values1, Object[] values2, Object[] result) {
+ assert values1.length == values2.length && values2.length == descLength && values1.length == result.length;
+
+ for (int i = 0; i < descLength; i++) {
+ result[i] = aggs[i].aggregate(values1[i], values2[i]);
+ }
+
+ }
+
+ public void aggregate(Object[] values1, Object[] values2, Object[] result, boolean[] aggrMask) {
+ assert values1.length == values2.length && values2.length == descLength && values1.length == result.length && result.length == aggrMask.length;
+ for (int i = 0; i < descLength; i++) {
+ if (aggrMask[i]) {
+ result[i] = aggs[i].aggregate(values1[i], values2[i]);
+ }
+ }
+ }
+
public void collectStates(Object[] states) {
for (int i = 0; i < descLength; i++) {
states[i] = aggs[i].getState();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
index edaf806..2d73e59 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureCodec.java
@@ -18,19 +18,19 @@
package org.apache.kylin.measure;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.MeasureDesc;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+
/**
* @author yangli9
*
*/
@SuppressWarnings({ "rawtypes" })
-public class MeasureCodec {
+public class MeasureCodec implements java.io.Serializable {
private int nMeasures;
private DataTypeSerializer[] serializers;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
index 0076252..26b7298 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureIngester.java
@@ -18,14 +18,14 @@
package org.apache.kylin.measure;
-import java.util.Collection;
-import java.util.Map;
-
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-abstract public class MeasureIngester<V> {
+import java.util.Collection;
+import java.util.Map;
+
+abstract public class MeasureIngester<V> implements java.io.Serializable {
public static MeasureIngester<?> create(MeasureDesc measure) {
return measure.getFunction().getMeasureType().newIngester();
@@ -42,6 +42,10 @@ abstract public class MeasureIngester<V> {
abstract public V valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap);
+ public void reset() {
+
+ }
+
public V reEncodeDictionary(V value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 89ff382..3338c8c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -18,11 +18,6 @@
package org.apache.kylin.measure;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -32,13 +27,18 @@ import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
/**
* MeasureType captures how a kind of aggregation is defined, how it is calculated
* during cube build, and how it is involved in query and storage scan.
*
* @param <T> the Java type of aggregation data object, e.g. HLLCounter
*/
-abstract public class MeasureType<T> {
+abstract public class MeasureType<T> implements java.io.Serializable {
/* ============================================================================
* Define
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
index aa42476..7a57965 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMaxAggregator.java
@@ -43,6 +43,20 @@ public class BigDecimalMaxAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ if (value1.compareTo(value2) > 0)
+ return value1;
+ else
+ return value2;
+ }
+
+ @Override
public BigDecimal getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
index 81193ad..71ba7fb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalMinAggregator.java
@@ -27,24 +27,38 @@ import org.apache.kylin.measure.MeasureAggregator;
@SuppressWarnings("serial")
public class BigDecimalMinAggregator extends MeasureAggregator<BigDecimal> {
- BigDecimal max = null;
+ BigDecimal min = null;
@Override
public void reset() {
- max = null;
+ min = null;
}
@Override
public void aggregate(BigDecimal value) {
- if (max == null)
- max = value;
- else if (max.compareTo(value) > 0)
- max = value;
+ if (min == null)
+ min = value;
+ else if (min.compareTo(value) > 0)
+ min = value;
+ }
+
+ @Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ if (value1.compareTo(value2) > 0)
+ return value2;
+ else
+ return value1;
}
@Override
public BigDecimal getState() {
- return max;
+ return min;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
index 5e00c63..9f6ffc2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java
@@ -40,6 +40,16 @@ public class BigDecimalSumAggregator extends MeasureAggregator<BigDecimal> {
}
@Override
+ public BigDecimal aggregate(BigDecimal value1, BigDecimal value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+ return value1.add(value2);
+ }
+
+ @Override
public BigDecimal getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index a2f3980..90ecb0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -22,25 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class DoubleIngester extends MeasureIngester<DoubleMutable> {
-
- // avoid repeated object creation
- private DoubleMutable current = new DoubleMutable();
+public class DoubleIngester extends MeasureIngester<Double> {
@Override
- public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Double valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- DoubleMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Double(0);
else
- l.set(Double.parseDouble(values[0]));
- return l;
+ return Double.parseDouble(values[0]);
+ }
+
+ @Override
+ public void reset() {
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
index 25911e8..f33555e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMaxAggregator extends MeasureAggregator<Double> {
- DoubleMutable max = null;
+ Double max = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (max == null)
- max = new DoubleMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Math.max(value1, value2);
+ }
+
+ @Override
+ public Double getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
index be97deb..8e69f21 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleMinAggregator extends MeasureAggregator<Double> {
- DoubleMutable min = null;
+ Double min = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> {
}
@Override
- public void aggregate(DoubleMutable value) {
+ public void aggregate(Double value) {
if (min == null)
- min = new DoubleMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Math.min(value1, value2);
+ }
+
+ @Override
+ public Double getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
index f276817..df0ba52 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java
@@ -19,27 +19,31 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
/**
*/
@SuppressWarnings("serial")
-public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> {
+public class DoubleSumAggregator extends MeasureAggregator<Double> {
- DoubleMutable sum = new DoubleMutable();
+ Double sum = new Double(0);
@Override
public void reset() {
- sum.set(0.0);
+ sum = new Double(0);
}
@Override
- public void aggregate(DoubleMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Double value) {
+ sum = sum + value;
}
@Override
- public DoubleMutable getState() {
+ public Double aggregate(Double value1, Double value2) {
+ return Double.valueOf(value1 + value2);
+ }
+
+ @Override
+ public Double getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index 45a1634..793acf2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -22,25 +22,24 @@ import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.measure.MeasureIngester;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-public class LongIngester extends MeasureIngester<LongMutable> {
-
- // avoid repeated object creation
- private LongMutable current = new LongMutable();
+public class LongIngester extends MeasureIngester<Long> {
@Override
- public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (values.length > 1)
throw new IllegalArgumentException();
- LongMutable l = current;
if (values[0] == null || values[0].length() == 0)
- l.set(0L);
+ return new Long(0L);
else
- l.set(Long.parseLong(values[0]));
- return l;
+ return Long.valueOf(values[0]);
+ }
+
+ @Override
+ public void reset() {
+
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
index ca44f15..b9a2b94 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
+public class LongMaxAggregator extends MeasureAggregator<Long> {
- LongMutable max = null;
+ Long max = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class LongMaxAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (max == null)
- max = new LongMutable(value.get());
- else if (max.get() < value.get())
- max.set(value.get());
+ max = value;
+ else if (max < value)
+ max = value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Math.max(value1, value2);
+ }
+
+ @Override
+ public Long getState() {
return max;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
index dadc64e..9185142 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java
@@ -19,14 +19,13 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongMinAggregator extends MeasureAggregator<LongMutable> {
+public class LongMinAggregator extends MeasureAggregator<Long> {
- LongMutable min = null;
+ Long min = null;
@Override
public void reset() {
@@ -34,15 +33,20 @@ public class LongMinAggregator extends MeasureAggregator<LongMutable> {
}
@Override
- public void aggregate(LongMutable value) {
+ public void aggregate(Long value) {
if (min == null)
- min = new LongMutable(value.get());
- else if (min.get() > value.get())
- min.set(value.get());
+ min = value;
+ else if (min > value)
+ min = value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Math.min(value1, value2);
+ }
+
+ @Override
+ public Long getState() {
return min;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
index e7fdc9d..1f9c0d7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java
@@ -19,27 +19,31 @@
package org.apache.kylin.measure.basic;
import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.metadata.datatype.LongMutable;
/**
*/
@SuppressWarnings("serial")
-public class LongSumAggregator extends MeasureAggregator<LongMutable> {
+public class LongSumAggregator extends MeasureAggregator<Long> {
- LongMutable sum = new LongMutable();
+ Long sum = new Long(0L);
@Override
public void reset() {
- sum.set(0);
+ sum = new Long(0L);
}
@Override
- public void aggregate(LongMutable value) {
- sum.set(sum.get() + value.get());
+ public void aggregate(Long value) {
+ sum += value;
}
@Override
- public LongMutable getState() {
+ public Long aggregate(Long value1, Long value2) {
+ return Long.valueOf(value1 + value2);
+ }
+
+ @Override
+ public Long getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
index be72090..cd0b4bb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapAggregator.java
@@ -42,6 +42,19 @@ public class BitmapAggregator extends MeasureAggregator<BitmapCounter> {
}
@Override
+ public BitmapCounter aggregate(BitmapCounter value1, BitmapCounter value2) {
+ if (value1 == null) {
+ return new BitmapCounter(value2);
+ } else if (value2 == null) {
+ return new BitmapCounter(value1);
+ }
+
+ BitmapCounter merged = new BitmapCounter(value1);
+ merged.merge(value2);
+ return merged;
+ }
+
+ @Override
public BitmapCounter getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a632b0d..43ee506 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -31,7 +31,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
/**
* Created by sunyerui on 15/12/1.
*/
-public class BitmapCounter implements Comparable<BitmapCounter> {
+public class BitmapCounter implements Comparable<BitmapCounter>, java.io.Serializable {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
private final int VERSION = 2;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index 8e2b2f7..6ad82a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -138,6 +138,11 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
}
return retValue;
}
+
+ @Override
+ public void reset() {
+ current = new BitmapCounter();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index 6fa8788..1b2cda3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -254,6 +254,21 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
}
@Override
+ public ByteArray aggregate(ByteArray value1, ByteArray value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ } else if (!value1.equals(value2)) {
+ if (!warned) {
+ logger.warn("Extended column must be unique given same host column");
+ warned = true;
+ }
+ }
+ return value1;
+ }
+
+ @Override
public ByteArray getState() {
return byteArray;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
index 5b929b2..4be9f71 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
@@ -24,7 +24,7 @@ import java.util.Map;
/**
* Created by xiefan on 16-12-9.
*/
-public class DenseRegister implements Register {
+public class DenseRegister implements Register, java.io.Serializable {
private int m;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
index 5966c04..7d5b216 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -46,6 +46,13 @@ public class HLLCAggregator extends MeasureAggregator<HLLCounter> {
}
@Override
+ public HLLCounter aggregate(HLLCounter value1, HLLCounter value2) {
+ HLLCounter result = new HLLCounter(value1);
+ result.merge(value2);
+ return result;
+ }
+
+ @Override
public HLLCounter getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 9601653..de36b08 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -105,6 +105,11 @@ public class HLLCMeasureType extends MeasureType<HLLCounter> {
}
return hllc;
}
+
+ @Override
+ public void reset() {
+ current = new HLLCounter(dataType.getPrecision());
+ }
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index e0992c7..df0cfaf 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -18,21 +18,18 @@
package org.apache.kylin.measure.hllc;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
/**
* @author yangli9
*
*/
public class HLLCSerializer extends DataTypeSerializer<HLLCounter> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<HLLCounter> current = new ThreadLocal<HLLCounter>();
-
private int precision;
public HLLCSerializer(DataType type) {
@@ -49,7 +46,7 @@ public class HLLCSerializer extends DataTypeSerializer<HLLCounter> {
}
private HLLCounter current() {
- HLLCounter hllc = current.get();
+ HLLCounter hllc = (HLLCounter) current.get();
if (hllc == null) {
hllc = new HLLCounter(precision);
current.set(hllc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
index 9d8de07..5d7bfeb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusTable.java
@@ -25,7 +25,7 @@ import java.util.TreeMap;
* @author xjiang
*
*/
-public class HyperLogLogPlusTable {
+public class HyperLogLogPlusTable implements java.io.Serializable {
// threshold and bias data taken from google's bias correction data set:
// https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
index 7f612e2..ad6306a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SingleValueRegister.java
@@ -20,7 +20,7 @@ package org.apache.kylin.measure.hllc;
/**
* Created by xiefan on 16-12-20.
*/
-public class SingleValueRegister implements Register {
+public class SingleValueRegister implements Register, java.io.Serializable {
private int singleValuePos;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
index dd7d7c8..d47a05b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
@@ -25,7 +25,7 @@ import java.util.TreeMap;
/**
* Created by xiefan on 16-12-9.
*/
-public class SparseRegister implements Register {
+public class SparseRegister implements Register, java.io.Serializable {
private Map<Integer, Byte> sparseRegister = new TreeMap<>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
index 6f66269..c3ecc74 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawAggregator.java
@@ -47,6 +47,20 @@ public class RawAggregator extends MeasureAggregator<List<ByteArray>> {
}
@Override
+ public List<ByteArray> aggregate(List<ByteArray> value1, List<ByteArray> value2) {
+ if (value1 == null) {
+ return value2;
+ } else if (value2 == null) {
+ return value1;
+ }
+
+ List<ByteArray> result = new ArrayList<>(value1.size() + value2.size());
+ result.addAll(value1);
+ result.addAll(value2);
+ return result;
+ }
+
+ @Override
public List<ByteArray> getState() {
return list;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
index 021c146..68a0273 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -34,13 +34,11 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
//FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file
public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M
- private ThreadLocal<List<ByteArray>> current = new ThreadLocal<>();
-
public RawSerializer(DataType dataType) {
}
private List<ByteArray> current() {
- List<ByteArray> l = current.get();
+ List<ByteArray> l = (List<ByteArray>) current.get();
if (l == null) {
l = new ArrayList<ByteArray>();
current.set(l);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
index 38d5b20..ac925e2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/DoubleDeltaSerializer.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
*
* http://bitcharmer.blogspot.co.uk/2013/12/how-to-serialise-array-of-doubles-with.html
*/
-public class DoubleDeltaSerializer {
+public class DoubleDeltaSerializer implements java.io.Serializable {
// first 32 bits stores meta info
static final int PRECISION_BITS = 3;
@@ -55,7 +55,6 @@ public class DoubleDeltaSerializer {
this.precision = precision;
this.multiplier = (int) Math.pow(10, precision);
- this.deltasThreadLocal = new ThreadLocal<long[]>();
}
public void serialize(double[] values, ByteBuffer buf) {
@@ -112,6 +111,10 @@ public class DoubleDeltaSerializer {
int len = values.length - 1;
len = Math.max(0, len);
+ if (deltasThreadLocal == null) {
+ deltasThreadLocal = new ThreadLocal<>();
+ }
+
long[] deltas = deltasThreadLocal.get();
if (deltas == null || deltas.length < len) {
deltas = new long[len];
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
index ef997eb..b5e316f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNAggregator.java
@@ -45,6 +45,15 @@ public class TopNAggregator extends MeasureAggregator<TopNCounter<ByteArray>> {
}
@Override
+ public TopNCounter<ByteArray> aggregate(TopNCounter<ByteArray> value1, TopNCounter<ByteArray> value2) {
+ TopNCounter<ByteArray> aggregated = new TopNCounter<>(capacity * 2);
+ aggregated.merge(value1);
+ aggregated.merge(value2);
+ aggregated.retain(capacity);
+ return aggregated;
+ }
+
+ @Override
public TopNCounter<ByteArray> getState() {
sum.retain(capacity);
return sum;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index caf7961..5e4b91e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -40,7 +40,7 @@ import com.google.common.collect.Maps;
*
* @param <T> type of data in the stream to be summarized
*/
-public class TopNCounter<T> implements Iterable<Counter<T>> {
+public class TopNCounter<T> implements Iterable<Counter<T>>, java.io.Serializable {
public static final int EXTRA_SPACE_RATE = 50;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index c29af6c..8c8b5a6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
@@ -46,8 +47,6 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
-
public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
@@ -156,6 +155,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return topNCounter;
}
+
@Override
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
index 64968b8..b5043f5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java
@@ -35,7 +35,7 @@ public class BigDecimalSerializer extends DataTypeSerializer<BigDecimal> {
private static final Logger logger = LoggerFactory.getLogger(BigDecimalSerializer.class);
final DataType type;
- transient final int maxLength;
+ final int maxLength;
transient int avoidVerbose = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
index acb6de1..3d485d2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java
@@ -18,40 +18,26 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.BooleanUtils;
-public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
+import java.nio.ByteBuffer;
- public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };
+public class BooleanSerializer extends DataTypeSerializer<Long> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+ public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" };
public BooleanSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -70,12 +56,10 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- LongMutable l = current();
+ public Long valueOf(String str) {
if (str == null)
- l.set(0L);
+ return Long.valueOf(0L);
else
- l.set(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
- return l;
+ return Long.valueOf(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase())));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
index a739377..a4a35a4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataTypeSerializer.java
@@ -18,19 +18,21 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.BytesSerializer;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.nio.ByteBuffer;
+import java.util.Map;
/**
* Note: the implementations MUST be thread-safe.
*/
-abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
+abstract public class DataTypeSerializer<T> implements BytesSerializer<T>, java.io.Serializable {
final static Map<String, Class<?>> implementations = Maps.newHashMap();
+ protected transient ThreadLocal current = new ThreadLocal();
static {
implementations.put("char", StringSerializer.class);
implementations.put("varchar", StringSerializer.class);
@@ -94,4 +96,9 @@ abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {
else
return value.toString();
}
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ current = new ThreadLocal();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
index 07f98b3..5101766 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java
@@ -18,37 +18,23 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.DateFormat;
-public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
+import java.nio.ByteBuffer;
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class DateTimeSerializer extends DataTypeSerializer<Long> {
public DateTimeSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- out.putLong(value.get());
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ out.putLong(value);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(in.getLong());
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return in.getLong();
}
@Override
@@ -67,8 +53,8 @@ public class DateTimeSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(DateFormat.stringToMillis(str));
+ public Long valueOf(String str) {
+ return Long.valueOf(DateFormat.stringToMillis(str));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
index 976dc51..20cfff3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java
@@ -22,33 +22,19 @@ import java.nio.ByteBuffer;
/**
*/
-public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<DoubleMutable> current = new ThreadLocal<DoubleMutable>();
+public class DoubleSerializer extends DataTypeSerializer<Double> {
public DoubleSerializer(DataType type) {
}
@Override
- public void serialize(DoubleMutable value, ByteBuffer out) {
- out.putDouble(value.get());
- }
-
- private DoubleMutable current() {
- DoubleMutable d = current.get();
- if (d == null) {
- d = new DoubleMutable();
- current.set(d);
- }
- return d;
+ public void serialize(Double value, ByteBuffer out) {
+ out.putDouble(value);
}
@Override
- public DoubleMutable deserialize(ByteBuffer in) {
- DoubleMutable d = current();
- d.set(in.getDouble());
- return d;
+ public Double deserialize(ByteBuffer in) {
+ return in.getDouble();
}
@Override
@@ -67,7 +53,7 @@ public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> {
}
@Override
- public DoubleMutable valueOf(String str) {
- return new DoubleMutable(Double.parseDouble(str));
+ public Double valueOf(String str) {
+ return Double.parseDouble(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
index 7b95505..0e82e11 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java
@@ -18,39 +18,25 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class Int4Serializer extends DataTypeSerializer<IntMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<IntMutable> current = new ThreadLocal<IntMutable>();
+public class Int4Serializer extends DataTypeSerializer<Integer> {
public Int4Serializer(DataType type) {
}
@Override
- public void serialize(IntMutable value, ByteBuffer out) {
- BytesUtil.writeUnsigned(value.get(), 4, out);
- }
-
- private IntMutable current() {
- IntMutable l = current.get();
- if (l == null) {
- l = new IntMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Integer value, ByteBuffer out) {
+ BytesUtil.writeUnsigned(value, 4, out);
}
@Override
- public IntMutable deserialize(ByteBuffer in) {
- IntMutable l = current();
- l.set(BytesUtil.readUnsigned(in, 4));
- return l;
+ public Integer deserialize(ByteBuffer in) {
+ return BytesUtil.readUnsigned(in, 4);
}
@Override
@@ -69,7 +55,7 @@ public class Int4Serializer extends DataTypeSerializer<IntMutable> {
}
@Override
- public IntMutable valueOf(String str) {
- return new IntMutable(Integer.parseInt(str));
+ public Integer valueOf(String str) {
+ return Integer.parseInt(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
index fa333b2..7dd5aa7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java
@@ -18,39 +18,26 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class Long8Serializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class Long8Serializer extends DataTypeSerializer<Long> {
public Long8Serializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeLong(value.get(), out);
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeLong(value, out);
}
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
- }
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readLong(in);
}
@Override
@@ -69,7 +56,7 @@ public class Long8Serializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
index 9306a70..605dcd7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java
@@ -18,39 +18,25 @@
package org.apache.kylin.metadata.datatype;
-import java.nio.ByteBuffer;
-
import org.apache.kylin.common.util.BytesUtil;
+import java.nio.ByteBuffer;
+
/**
*/
-public class LongSerializer extends DataTypeSerializer<LongMutable> {
-
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<LongMutable> current = new ThreadLocal<LongMutable>();
+public class LongSerializer extends DataTypeSerializer<Long> {
public LongSerializer(DataType type) {
}
@Override
- public void serialize(LongMutable value, ByteBuffer out) {
- BytesUtil.writeVLong(value.get(), out);
- }
-
- private LongMutable current() {
- LongMutable l = current.get();
- if (l == null) {
- l = new LongMutable();
- current.set(l);
- }
- return l;
+ public void serialize(Long value, ByteBuffer out) {
+ BytesUtil.writeVLong(value, out);
}
@Override
- public LongMutable deserialize(ByteBuffer in) {
- LongMutable l = current();
- l.set(BytesUtil.readVLong(in));
- return l;
+ public Long deserialize(ByteBuffer in) {
+ return BytesUtil.readVLong(in);
}
@Override
@@ -75,7 +61,7 @@ public class LongSerializer extends DataTypeSerializer<LongMutable> {
}
@Override
- public LongMutable valueOf(String str) {
- return new LongMutable(Long.parseLong(str));
+ public Long valueOf(String str) {
+ return Long.parseLong(str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 184c290..a49d982 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,27 +18,27 @@
package org.apache.kylin.metadata.model;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class FunctionDesc {
+public class FunctionDesc implements Serializable {
public static FunctionDesc newInstance(String expression, ParameterDesc param, String returnType) {
FunctionDesc r = new FunctionDesc();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 6489244..dd1500b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -18,17 +18,18 @@
package org.apache.kylin.metadata.model;
-import java.util.Arrays;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
+import java.io.Serializable;
+import java.util.Arrays;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JoinDesc {
+public class JoinDesc implements Serializable {
// inner, left, right, outer...
@JsonProperty("type")
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 513217e..51e5787 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -28,7 +28,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JoinTableDesc implements Serializable{
+public class JoinTableDesc implements Serializable {
@JsonProperty("table")
private String table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index a0b267d..c132d0e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -18,15 +18,16 @@
package org.apache.kylin.metadata.model;
+import com.google.common.base.Preconditions;
+
+import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Preconditions;
-
-public class JoinsTree {
+public class JoinsTree implements Serializable {
final Map<String, Chain> tableChains = new LinkedHashMap<>();
@@ -111,7 +112,7 @@ public class JoinsTree {
return chain.join;
}
- static class Chain {
+ static class Chain implements java.io.Serializable {
TableRef table; // pk side
JoinDesc join;
Chain fkSide;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 6ddbbf4..deec4f2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -22,15 +22,15 @@ import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class MeasureDesc implements Serializable{
+public class MeasureDesc implements Serializable {
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index bd3dfef..d14a56b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ModelDimensionDesc implements Serializable{
+public class ModelDimensionDesc implements Serializable {
@JsonProperty("table")
private String table;
@JsonProperty("columns")
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index c14d061..8ad20a8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -18,21 +18,22 @@
package org.apache.kylin.metadata.model;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.List;
-
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class ParameterDesc {
+public class ParameterDesc implements Serializable {
public static ParameterDesc newInstance(Object... objs) {
if (objs.length == 0)
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index bab3ec3..c6e6425 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -32,7 +32,7 @@ import java.io.Serializable;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class PartitionDesc implements Serializable{
+public class PartitionDesc implements Serializable {
public static enum PartitionType {
APPEND, //
@@ -177,7 +177,7 @@ public class PartitionDesc implements Serializable{
String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive);
}
- public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder {
+ public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder, java.io.Serializable {
@Override
public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 9837ff4..6f15f3c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -18,26 +18,28 @@
package org.apache.kylin.metadata.model;
+import com.google.common.collect.Maps;
+
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import com.google.common.collect.Maps;
-
public class TableRef implements Serializable{
- final private DataModelDesc model;
+ final transient private DataModelDesc model;
final private String alias;
final private TableDesc table;
final private Map<String, TblColRef> columns;
+ final private String modelName;
TableRef(DataModelDesc model, String alias, TableDesc table) {
this.model = model;
+ this.modelName = model.getName();
this.alias = alias;
this.table = table;
this.columns = Maps.newLinkedHashMap();
-
+
for (ColumnDesc col : table.getColumns()) {
columns.put(col.getName(), new TblColRef(this, col));
}
@@ -95,7 +97,7 @@ public class TableRef implements Serializable{
TableRef t = (TableRef) o;
- if ((model == null ? t.model == null : model.getName().equals(t.model.getName())) == false)
+ if ((modelName == null ? t.modelName != null : modelName.equals(t.modelName)) == false)
return false;
if ((alias == null ? t.alias == null : alias.equals(t.alias)) == false)
return false;
@@ -108,7 +110,7 @@ public class TableRef implements Serializable{
@Override
public int hashCode() {
int result = 0;
- result = 31 * result + model.getName().hashCode();
+ result = 31 * result + modelName.hashCode();
result = 31 * result + alias.hashCode();
result = 31 * result + table.getIdentity().hashCode();
return result;
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 40e5b29..3d48ac2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -38,8 +38,6 @@ import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCAggregator;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.DoubleMutable;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -61,7 +59,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
}
private List<? extends MeasureAggregator> basicAggregators() {
- LongMutable longVal = new LongMutable(1000);
+ Long longVal = new Long(1000);
LongMinAggregator longMin = new LongMinAggregator();
LongMaxAggregator longMax = new LongMaxAggregator();
LongSumAggregator longSum = new LongSumAggregator();
@@ -69,7 +67,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
longMax.aggregate(longVal);
longSum.aggregate(longVal);
- DoubleMutable doubleVal = new DoubleMutable(1.0);
+ Double doubleVal = new Double(1.0);
DoubleMinAggregator doubleMin = new DoubleMinAggregator();
DoubleMaxAggregator doubleMax = new DoubleMaxAggregator();
DoubleSumAggregator doubleSum = new DoubleSumAggregator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index dde0481..fcd0182 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -52,7 +52,6 @@ import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.memstore.GTSimpleMemStore;
import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -499,16 +498,16 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
GTRecord r = new GTRecord(table.getInfo());
GTBuilder builder = table.rebuild();
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
builder.close();
return table;
@@ -524,34 +523,34 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
for (int i = 0; i < 100000; i++) {
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5")));
for (int j = 0; j < 10; j++)
- builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5")));
+ builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5")));
}
builder.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 1e3f078..700f821 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -31,6 +31,7 @@ import org.apache.kylin.engine.mr.steps.NDCuboidJob;
import org.apache.kylin.engine.mr.steps.SaveStatisticsStep;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.execution.AbstractExecutable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,7 +97,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return result;
}
- private MapReduceExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
+ protected AbstractExecutable createInMemCubingStep(String jobId, String cuboidRootPath) {
// base cuboid job
MapReduceExecutable cubeStep = new MapReduceExecutable();
@@ -113,7 +114,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
cubeStep.setMapReduceParams(cmd.toString());
cubeStep.setMapReduceJobClass(getInMemCuboidJob());
- cubeStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
+// cubeStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
return cubeStep;
}
@@ -140,7 +141,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
baseCuboidStep.setMapReduceParams(cmd.toString());
baseCuboidStep.setMapReduceJobClass(getBaseCuboidJob());
- baseCuboidStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
+// baseCuboidStep.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
return baseCuboidStep;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 3119c1e..7665350 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
public class HadoopUtil {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HadoopUtil.class);
- private static final ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
+ private static final transient ThreadLocal<Configuration> hadoopConfig = new ThreadLocal<>();
public static void setCurrentConfiguration(Configuration conf) {
hadoopConfig.set(conf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b7d87bb4/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 86ac880..14252ee 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -75,6 +75,7 @@ public class JobBuilderSupport {
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Fact_Distinct_Columns_" + seg.getRealization().getName() + "_Step");
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
result.setMapReduceParams(cmd.toString());
+ result.setCounterSaveAs(CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES);
return result;
}
@@ -204,4 +205,13 @@ public class JobBuilderSupport {
return paths;
}
+ public static String getCuboidOutputPathsByLevel(String cuboidRootPath, int level) {
+ if (level == 0) {
+ return cuboidRootPath + "base_cuboid";
+ } else {
+ return cuboidRootPath + level + "level_cuboid";
+ }
+ }
+
+
}
[19/50] [abbrv] kylin git commit: KYLIN-2337 minor improvement,
still give good error info
Posted by li...@apache.org.
KYLIN-2337 minor improvement, still give good error info
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/122b15a2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/122b15a2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/122b15a2
Branch: refs/heads/master-hbase1.x
Commit: 122b15a2a5fb7385e64839edfb9cbdaec5fde841
Parents: 7509bee
Author: Yang Li <li...@apache.org>
Authored: Sat Jan 7 09:38:06 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 09:38:06 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/122b15a2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
index 6a7e713..c5c95a6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortedIteratorMergerWithLimit.java
@@ -126,7 +126,8 @@ public class SortedIteratorMergerWithLimit<E extends Cloneable> extends SortedIt
//TODO: remove this check when validated
if (last != null) {
- Preconditions.checkState(comparator.compare(last, fetched) <= 0, "Not sorted!");
+ if (comparator.compare(last, fetched) > 0)
+ throw new IllegalStateException("Not sorted! last: " + last + " fetched: " + fetched);
}
last = fetched;
[45/50] [abbrv] kylin git commit: minor,
change Dictionary to Dictionary in DictionaryManager and
DictionaryInfo
Posted by li...@apache.org.
minor,change Dictionary<?> to Dictionary<String> in DictionaryManager and DictionaryInfo
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/be11dc62
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/be11dc62
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/be11dc62
Branch: refs/heads/master-hbase1.x
Commit: be11dc629460ef4620300e1508c548037540f961
Parents: 5f6450d
Author: xiefan46 <95...@qq.com>
Authored: Tue Jan 10 16:33:25 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 11 13:04:36 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/DictionaryInfo.java | 6 +++---
.../apache/kylin/dict/DictionaryInfoSerializer.java | 4 ++--
.../java/org/apache/kylin/dict/DictionaryManager.java | 14 +++++++-------
.../kylin/engine/mr/steps/MergeCuboidMapperTest.java | 6 +++---
4 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
index 8526467..a85628d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfo.java
@@ -47,7 +47,7 @@ public class DictionaryInfo extends RootPersistentEntity {
@JsonProperty("cardinality")
private int cardinality;
- transient Dictionary<?> dictionaryObject;
+ transient Dictionary<String> dictionaryObject;
public DictionaryInfo() {
}
@@ -144,11 +144,11 @@ public class DictionaryInfo extends RootPersistentEntity {
this.dictionaryClass = dictionaryClass;
}
- public Dictionary<?> getDictionaryObject() {
+ public Dictionary<String> getDictionaryObject() {
return dictionaryObject;
}
- public void setDictionaryObject(Dictionary<?> dictionaryObject) {
+ public void setDictionaryObject(Dictionary<String> dictionaryObject) {
this.dictionaryObject = dictionaryObject;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
index 04eedbe..ee88ea7 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryInfoSerializer.java
@@ -61,9 +61,9 @@ public class DictionaryInfoSerializer implements Serializer<DictionaryInfo> {
DictionaryInfo obj = JsonUtil.readValue(json, DictionaryInfo.class);
if (infoOnly == false) {
- Dictionary<?> dict;
+ Dictionary<String> dict;
try {
- dict = (Dictionary<?>) ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
+ dict = (Dictionary<String>) ClassUtil.forName(obj.getDictionaryClass(), Dictionary.class).newInstance();
} catch (InstantiationException e) {
throw new RuntimeException(e);
} catch (IllegalAccessException e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 54bc1c4..3ba24cf 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -108,7 +108,7 @@ public class DictionaryManager {
});
}
- public Dictionary<?> getDictionary(String resourcePath) throws IOException {
+ public Dictionary<String> getDictionary(String resourcePath) throws IOException {
DictionaryInfo dictInfo = getDictionaryInfo(resourcePath);
return dictInfo == null ? null : dictInfo.getDictionaryObject();
}
@@ -130,7 +130,7 @@ public class DictionaryManager {
* Save the dictionary as it is.
* More often you should consider using its alternative trySaveNewDict to save dict space
*/
- public DictionaryInfo forceSave(Dictionary<?> newDict, DictionaryInfo newDictInfo) throws IOException {
+ public DictionaryInfo forceSave(Dictionary<String> newDict, DictionaryInfo newDictInfo) throws IOException {
initDictInfo(newDict, newDictInfo);
logger.info("force to save dict directly");
return saveNewDict(newDictInfo);
@@ -140,7 +140,7 @@ public class DictionaryManager {
* @return may return another dict that is a super set of the input
* @throws IOException
*/
- public DictionaryInfo trySaveNewDict(Dictionary<?> newDict, DictionaryInfo newDictInfo) throws IOException {
+ public DictionaryInfo trySaveNewDict(Dictionary<String> newDict, DictionaryInfo newDictInfo) throws IOException {
initDictInfo(newDict, newDictInfo);
@@ -149,7 +149,7 @@ public class DictionaryManager {
DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
if (largestDictInfo != null) {
largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
- Dictionary<?> largestDictObject = largestDictInfo.getDictionaryObject();
+ Dictionary<String> largestDictObject = largestDictInfo.getDictionaryObject();
if (largestDictObject.contains(newDict)) {
logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath());
return largestDictInfo;
@@ -175,7 +175,7 @@ public class DictionaryManager {
}
}
- private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
+ private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException {
ResourceStore store = MetadataManager.getInstance(config).getStore();
NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir());
if (existings == null)
@@ -196,7 +196,7 @@ public class DictionaryManager {
return null;
}
- private void initDictInfo(Dictionary<?> newDict, DictionaryInfo newDictInfo) {
+ private void initDictInfo(Dictionary<String> newDict, DictionaryInfo newDictInfo) {
newDictInfo.setCardinality(newDict.getSize());
newDictInfo.setDictionaryObject(newDict);
newDictInfo.setDictionaryClass(newDict.getClass().getName());
@@ -263,7 +263,7 @@ public class DictionaryManager {
logger.info("Use one of the merging dictionaries directly");
return dicts.get(0);
} else {
- Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts);
+ Dictionary<String> newDict = DictionaryGenerator.mergeDictionaries(DataType.getType(newDictInfo.getDataType()), dicts);
return trySaveNewDict(newDict, newDictInfo);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/be11dc62/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index 075ec80..dc593b0 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -75,7 +75,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
List<String> values = new ArrayList<>();
values.add("eee");
values.add("fff");
- Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
+ Dictionary<String> dict = DictionaryGenerator.buildDictionary(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
dictionaryManager.trySaveNewDict(dict, newDictInfo);
dict.dump(System.out);
@@ -127,7 +127,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
values.add("ccc");
else
values.add("bbb");
- Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
+ Dictionary<String> dict = DictionaryGenerator.buildDictionary(DataType.getType(newDictInfo.getDataType()), new IterableDictionaryValueEnumerator(values));
dictionaryManager.trySaveNewDict(dict, newDictInfo);
dict.dump(System.out);
@@ -161,7 +161,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, Long.MAX_VALUE, 0, 0, false);
// String segmentName = newSeg.getName();
- final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn);
+ final Dictionary<String> dictionary = cubeManager.getDictionary(newSeg, lfn);
assertTrue(dictionary == null);
// ((TrieDictionary) dictionary).dump(System.out);
[26/50] [abbrv] kylin git commit: minor, enable jacoco plugin
Posted by li...@apache.org.
minor, enable jacoco plugin
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/68ff75de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/68ff75de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/68ff75de
Branch: refs/heads/master-hbase1.x
Commit: 68ff75de260e6e93197103c0fd06009cf73c8e21
Parents: e46d699
Author: Billy Liu <bi...@apache.org>
Authored: Mon Jan 9 00:29:57 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Mon Jan 9 00:29:57 2017 +0800
----------------------------------------------------------------------
pom.xml | 47 +++++++++++++++++++++++++++++++----------------
1 file changed, 31 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/68ff75de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 22681e8..d883eca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -922,19 +922,7 @@
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
- <version>0.7.7.201606060606</version>
- <configuration>
- <destFile>${sonar.jacoco.reportPath}</destFile>
- <append>true</append>
- </configuration>
- <executions>
- <execution>
- <id>agent</id>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- </execution>
- </executions>
+ <version>0.7.8</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -1079,6 +1067,35 @@
</plugin>
<plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <configuration>
+ <append>true</append>
+ <destFile>
+ ${sonar.jacoco.reportPath}
+ </destFile>
+ </configuration>
+ <executions>
+ <execution>
+ <id>pre-test</id>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ <configuration>
+ <propertyName>surefireArgLine</propertyName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>post-test</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
@@ -1101,11 +1118,9 @@
<value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
</property>
</systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine}</argLine>
+ <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
</configuration>
</plugin>
-
-
</plugins>
</build>
</profile>
[37/50] [abbrv] kylin git commit: KYLIN-2371 rename the prefix to ‘kylin.engine.spark-conf.’
Posted by li...@apache.org.
KYLIN-2371 rename the prefix to \u2018kylin.engine.spark-conf.\u2019
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00c8f31e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00c8f31e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00c8f31e
Branch: refs/heads/master-hbase1.x
Commit: 00c8f31e6d235d38bb7919fa4ee5b12e53d840f6
Parents: ee74a74
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jan 10 09:40:44 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Jan 10 09:40:44 2017 +0800
----------------------------------------------------------------------
build/conf/kylin-spark-conf.properties | 27 -----------
build/conf/kylin.properties | 48 ++++++++++++--------
build/deploy/spark-defaults.conf | 6 +++
build/script/download-spark.sh | 3 +-
.../apache/kylin/common/KylinConfigBase.java | 18 +-------
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 2 +-
.../kylin/engine/spark/SparkCubingByLayer.java | 10 ++--
.../kylin/engine/spark/SparkExecutable.java | 7 ++-
.../test_case_data/sandbox/kylin.properties | 19 ++++++--
9 files changed, 65 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/conf/kylin-spark-conf.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties
deleted file mode 100644
index 143e7e4..0000000
--- a/build/conf/kylin-spark-conf.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-spark.yarn.submit.file.replication=1
-spark.yarn.executor.memoryOverhead=1024
-spark.yarn.driver.memoryOverhead=384
-spark.master=yarn
-spark.submit.deployMode=cluster
-spark.eventLog.enabled=true
-spark.yarn.scheduler.heartbeat.interval-ms=5000
-spark.yarn.preserve.staging.files=true
-spark.yarn.queue=default
-spark.yarn.containerLauncherMaxThreads=25
-spark.yarn.max.executor.failures=3
-spark.eventLog.dir=hdfs\:///kylin/spark-history
-spark.history.kerberos.enabled=true
-spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider
-spark.history.ui.port=18080
-spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
-spark.executor.memory=4G
-spark.storage.memoryFraction=0.3
-spark.executor.cores=4
-spark.executor.instances=8
-spark.history.kerberos.keytab=none
-spark.history.kerberos.principal=none
-spark.driver.extraJavaOptions=-Dhdp.version=current
-spark.yarn.am.extraJavaOptions=-Dhdp.version=current
-spark.executor.extraJavaOptions=-Dhdp.version=current
-#spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
-#spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 6efa423..196a711 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -127,24 +127,6 @@ kylin.engine.mr.max-reducer-number=500
kylin.engine.mr.mapper-input-rows=1000000
-
-### Spark Engine ###
-
-# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
-#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
-
-# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties
-#kylin.engine.spark.properties-file=
-
-# Estimate the RDD partition numbers
-kylin.engine.spark.rdd-partition-cut-mb=10
-
-# Minimal partition numbers of rdd
-kylin.engine.spark.min-partition=1
-
-# Max partition numbers of rdd
-kylin.engine.spark.max-partition=5000
-
### CUBE | DICTIONARY ###
# 'auto', 'inmem', 'layer' or 'random' for testing
@@ -204,3 +186,33 @@ kylin.security.saml.context-server-name=hostname
kylin.security.saml.context-server-port=443
kylin.security.saml.context-path=/kylin
+
+### Spark Engine Configs ###
+
+# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit
+#kylin.engine.spark.env.hadoop-conf-dir=/etc/hive/conf
+
+# Estimate the RDD partition numbers
+kylin.engine.spark.rdd-partition-cut-mb=10
+
+# Minimal partition numbers of rdd
+kylin.engine.spark.min-partition=1
+
+# Max partition numbers of rdd
+kylin.engine.spark.max-partition=5000
+
+### Spark conf (default is in spark/conf/spark-defaults.conf)
+kylin.engine.spark-conf.spark.master=yarn
+kylin.engine.spark-conf.spark.submit.deployMode=cluster
+kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024
+kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384
+kylin.engine.spark-conf.spark.yarn.queue=default
+kylin.engine.spark-conf.spark.executor.memory=4G
+kylin.engine.spark-conf.spark.executor.cores=4
+kylin.engine.spark-conf.spark.executor.instances=8
+kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
+#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/deploy/spark-defaults.conf
----------------------------------------------------------------------
diff --git a/build/deploy/spark-defaults.conf b/build/deploy/spark-defaults.conf
new file mode 100644
index 0000000..36c0ab3
--- /dev/null
+++ b/build/deploy/spark-defaults.conf
@@ -0,0 +1,6 @@
+spark.yarn.submit.file.replication=1
+spark.eventLog.enabled=true
+spark.yarn.max.executor.failures=3
+spark.driver.extraJavaOptions=-Dhdp.version=current
+spark.yarn.am.extraJavaOptions=-Dhdp.version=current
+spark.executor.extraJavaOptions=-Dhdp.version=current
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/build/script/download-spark.sh
----------------------------------------------------------------------
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
index dcbcbe7..ad9651d 100755
--- a/build/script/download-spark.sh
+++ b/build/script/download-spark.sh
@@ -48,5 +48,6 @@ mv build/spark-${spark_version}-bin-hadoop2.6 build/spark
rm -rf build/spark/lib/spark-examples-*
rm -rf build/spark/examples
rm -rf build/spark/data
-rm -rf build/spark/python
rm -rf build/spark/R
+
+cp build/deploy/spark-defaults.conf build/spark/conf/spark-defaults.conf
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 36ddbf4..04051b4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -696,7 +696,7 @@ abstract public class KylinConfigBase implements Serializable {
}
public Map<String, String> getSparkConfigOverride() {
- return getPropertiesByPrefix("kylin.engine.spark.config-override.");
+ return getPropertiesByPrefix("kylin.engine.spark-conf.");
}
public double getDefaultHadoopJobReducerInputMB() {
@@ -747,22 +747,6 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.engine.spark.env.hadoop-conf-dir", "");
}
- public String getSparkConfFile() {
- String conf = getOptional("kylin.engine.spark.properties-file", "conf/kylin-spark-conf.properties");
- File f = new File(conf);
- if (f.exists()) {
- return f.getAbsolutePath();
- } else {
- String home = getKylinHome();
- f = new File(home, conf);
- if (f.exists()) {
- return f.getAbsolutePath();
- }
- }
-
- throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist.");
- }
-
public String getSparkAdditionalJars() {
return getOptional("kylin.engine.spark.additional-jars", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index dd866bd..0f604e2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -82,7 +82,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
result.addTask(createBaseCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, 0), jobId));
// n dim cuboid steps
for (int i = 1; i <= maxLevel; i++) {
- result.addTask(createNDimensionCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, i-1), getCuboidOutputPathsByLevel(cuboidRootPath, i-1), i, jobId));
+ result.addTask(createNDimensionCuboidStep(getCuboidOutputPathsByLevel(cuboidRootPath, i-1), getCuboidOutputPathsByLevel(cuboidRootPath, i), i, jobId));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index c989dee..3a664fc 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -255,7 +255,10 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
// aggregate to calculate base cuboid
allRDDs[0] = encodedBaseRDD.reduceByKey(baseCuboidReducerFunction, partition).persist(storageLevel);
- saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, sc.hadoopConfiguration());
+ Configuration confOverwrite = new Configuration(sc.hadoopConfiguration());
+ confOverwrite.set("dfs.replication", "2"); // cuboid intermediate files, replication=2
+
+ saveToHDFS(allRDDs[0], vCubeDesc.getValue(), outputPath, 0, confOverwrite);
// aggregate to ND cuboids
PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> flatMapFunction = new CuboidFlatMap(vCubeSegment.getValue(), vCubeDesc.getValue(), vCuboidScheduler.getValue(), ndCuboidBuilder);
@@ -267,7 +270,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
if (kylinConfig.isSparkSanityCheckEnabled() == true) {
sanityCheck(allRDDs[level], totalCount, level, cubeStatsReader, countMeasureIndex);
}
- saveToHDFS(allRDDs[level], vCubeDesc.getValue(), outputPath, level, sc.hadoopConfiguration());
+ saveToHDFS(allRDDs[level], vCubeDesc.getValue(), outputPath, level, confOverwrite);
allRDDs[level - 1].unpersist();
}
allRDDs[totalLevels - 1].unpersist();
@@ -285,8 +288,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
private static void saveToHDFS(final JavaPairRDD<ByteArray, Object[]> rdd, final CubeDesc cubeDesc, final String hdfsBaseLocation, int level, Configuration conf) {
- conf.set("dfs.replication", "2");
- final String cuboidOutputPath = BatchCubingJobBuilder2.getCuboidOutputPathsByLevel(hdfsBaseLocation, level);
+ final String cuboidOutputPath = BatchCubingJobBuilder2.getCuboidOutputPathsByLevel(hdfsBaseLocation, level);
rdd.mapToPair(new PairFunction<Tuple2<ByteArray, Object[]>, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text>() {
BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index d892060..733a472 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -76,7 +76,6 @@ public class SparkExecutable extends AbstractExecutable {
final KylinConfig config = context.getConfig();
Preconditions.checkNotNull(config.getSparkHome());
Preconditions.checkNotNull(config.getKylinJobJarPath());
- String sparkConf = config.getSparkConfFile();
String jars = this.getParam(JARS);
String hadoopConf = "/etc/hadoop/conf";
@@ -105,17 +104,17 @@ public class SparkExecutable extends AbstractExecutable {
}
StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry --properties-file %s ");
+ stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry ");
Map<String, String> sparkConfs = config.getSparkConfigOverride();
for (Map.Entry<String, String> entry : sparkConfs.entrySet()) {
- stringBuilder.append(" --conf ").append(entry.getKey()).append("==").append(entry.getValue()).append(" ");
+ stringBuilder.append(" --conf ").append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
}
stringBuilder.append("--files %s --jars %s %s %s");
try {
String cmd = String.format(stringBuilder.toString(),
- hadoopConf, config.getSparkHome(), sparkConf, hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
+ hadoopConf, config.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs());
logger.info("cmd:" + cmd);
final StringBuilder output = new StringBuilder();
CliCommandExecutor exec = new CliCommandExecutor();
http://git-wip-us.apache.org/repos/asf/kylin/blob/00c8f31e/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index d42e009..06f8e4b 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -160,6 +160,19 @@ kylin.source.hive.keep-flat-table=false
### Spark as Engine ###
kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox
-kylin.engine.spark.spark-home=/usr/local/spark
-kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties
-kylin.engine.spark.sanity-check-enabled=false
\ No newline at end of file
+kylin.engine.spark.sanity-check-enabled=false
+
+### Spark conf overwrite for cube engine
+kylin.engine.spark-conf.spark.master=yarn
+kylin.engine.spark-conf.spark.submit.deployMode=client
+kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=512
+kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384
+kylin.engine.spark-conf.spark.executor.memory=1G
+kylin.engine.spark-conf.spark.executor.cores=1
+kylin.engine.spark-conf.spark.executor.instances=1
+kylin.engine.spark-conf.spark.storage.memoryFraction=0.3
+kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
+#kylin.engine.spark-conf.spark.yarn.queue=default
+#kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
+#kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec
[50/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c47f4ad5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c47f4ad5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c47f4ad5
Branch: refs/heads/master-hbase1.x
Commit: c47f4ad5767e7c9604b78bbae798805ce22e02fe
Parents: b0a406f
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Jan 12 12:02:26 2017 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 19 ++--
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 37 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 27 +++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
39 files changed, 366 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 62978db..e46cbe1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -32,11 +32,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,6 +56,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -92,10 +91,10 @@ public class BuildCubeWithEngine {
logger.error("error", e);
exitCode = 1;
}
-
+
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
-
+
System.exit(exitCode);
}
@@ -279,7 +278,7 @@ public class BuildCubeWithEngine {
String cubeName = "ci_inner_join_cube";
clearSegment(cubeName);
-
+
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long date1 = 0;
@@ -334,10 +333,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e391103..6ef99d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.1.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index ea68855..8095bf8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 1d520c4..d1a1384 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -58,11 +59,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index d28c87c..c8c87cb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -27,9 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -395,33 +393,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bc644cc..b07e698 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -49,11 +49,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -164,14 +164,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -197,14 +196,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -216,12 +214,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 2a12984..2a4a2f0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,14 +30,15 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -237,13 +238,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,8 +264,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -274,7 +272,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -287,7 +285,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -308,7 +305,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -316,7 +313,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -335,7 +332,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
@@ -360,7 +357,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -368,8 +365,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index df1817e..cad5a3f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -51,10 +52,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +118,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -172,7 +173,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 21a0efb..810747f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index fbe64d9..347807d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 31864f6..40e9fb8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -26,19 +26,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -89,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -233,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -326,8 +327,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -449,11 +450,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -473,7 +474,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -504,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -539,13 +539,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -562,7 +561,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 8bd4abf..20d0f7d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,9 +130,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -189,10 +188,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
[13/50] [abbrv] kylin git commit: KYLIN-2287 fix the button display
issue with js
Posted by li...@apache.org.
KYLIN-2287 fix the button display issue with js
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49de556f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49de556f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49de556f
Branch: refs/heads/master-hbase1.x
Commit: 49de556f00ff8fd854654daf1bc7e1d26c1fb879
Parents: f267ba9
Author: ShaoChinYu <sh...@kyligence.io>
Authored: Fri Jan 6 12:22:59 2017 +0800
Committer: zhongjian <ji...@163.com>
Committed: Fri Jan 6 12:29:35 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/models.js | 36 ++++++++++++++++++++++++
webapp/app/partials/models/models_tree.html | 6 ++--
2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/js/controllers/models.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/models.js b/webapp/app/js/controllers/models.js
index e3ee038..fd21845 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -183,7 +183,41 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
});
};
+ function changePositionOfScrollBar(){
+ //get which button be clicked
+ var btn = window.event.srcElement || window.event.target;
+ //get current position of scroll bar
+ var scrollTop =$("#cube_model_trees").scrollTop();
+ //get total length of scroll bar
+ var scrollHeight = document.getElementById('cube_model_trees').scrollHeight;
+ //get the position of clicked button relative to the top of window
+ var offsetTop =$(btn).offset().top;
+ //get the position of the container relative to the top of window
+ var treeOffsetTop = $("#cube_model_trees").offset().top;
+
+ //distance from button to the top of tree model container
+ var minor = offsetTop - treeOffsetTop;
+ //height of tree model container
+ var viewH =$("#cube_model_trees").height();
+
+ //change scroll bar to show the dropdown menu
+ if(minor + 100 > viewH){//100 is the height of dropdowm menu
+ if((scrollHeight - scrollTop - viewH)>=minor+100-viewH){
+ document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+ }else{
+ var node=document.createElement("LI");
+ node.style.height = (minor+120-viewH)+"px";
+ document.getElementById("models-tree").appendChild(node);
+ var viewH =$("#cube_model_trees").height();//\u53ef\u89c1\u9ad8\u5ea6
+ document.getElementById('cube_model_trees').scrollTop+=(minor+120-viewH);
+
+ }
+ }
+ }
+
$scope.listModelAccess = function (model) {
+ changePositionOfScrollBar();
+
if(model.uuid){
AccessService.list({type: "DataModelDesc", uuid: model.uuid}, function (accessEntities) {
model.accessEntities = accessEntities;
@@ -196,6 +230,8 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
}
})
}
+
+
};
var ModelDetailModalCtrl = function ($scope, $location, $modalInstance, scope) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/49de556f/webapp/app/partials/models/models_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/models_tree.html b/webapp/app/partials/models/models_tree.html
index 944d2dc..eba7d29 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -41,10 +41,10 @@
<div>
<h3 class="text-info">Models</h3>
</div>
+<!--{{window}}px -->
+ <div id="cube_model_trees" style="width:100%; height:250px; overflow:auto;margin-top: 20px;" class="cube_model_trees">
- <div style="width:100%; height:{{window}}px; overflow:auto;margin-top: 20px;" class="cube_model_trees">
-
- <ul class="list-group models-tree">
+ <ul class="list-group models-tree" id="models-tree">
<li class="list-group-item" ng-repeat="model in modelsManager.models">
<div class="pull-right" showonhoverparent style="display:none;" >
[21/50] [abbrv] kylin git commit: KYLIN-2338 refactor
BitmapCounter.DataInputByteBuffer
Posted by li...@apache.org.
KYLIN-2338 refactor BitmapCounter.DataInputByteBuffer
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/746e3801
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/746e3801
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/746e3801
Branch: refs/heads/master-hbase1.x
Commit: 746e3801faec58f0e10e619f0697ffbe37c08c63
Parents: 5a18af0
Author: kangkaisen <ka...@live.com>
Authored: Mon Dec 26 20:12:30 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:04:52 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 111 +++----------------
1 file changed, 18 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/746e3801/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 827390d..a7f277e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -26,7 +26,6 @@ import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import org.apache.commons.io.IOUtils;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
/**
@@ -107,12 +106,8 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
public void readRegisters(ByteBuffer in) throws IOException {
- DataInputByteBuffer input = new DataInputByteBuffer();
- try {
- input.reset(new ByteBuffer[] { in });
- bitmap.deserialize(input);
- } finally {
- IOUtils.closeQuietly(input);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ bitmap.deserialize(is);
}
}
@@ -178,15 +173,11 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
int mark = in.position();
int len;
- DataInputByteBuffer input = new DataInputByteBuffer();
- input.reset(new ByteBuffer[] { in });
MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
- try {
- bitmap.deserialize(input);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ bitmap.deserialize(is);
} catch (IOException e) {
throw new IllegalStateException(e);
- } finally {
- IOUtils.closeQuietly(input);
}
len = in.position() - mark;
@@ -194,93 +185,27 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return len;
}
- static class DataInputByteBuffer extends DataInputStream {
- private DataInputByteBuffer.Buffer buffers;
+ private class ByteBufferBackedInputStream extends InputStream {
+ private final ByteBuffer buffer;
- public DataInputByteBuffer() {
- this(new DataInputByteBuffer.Buffer());
+ private ByteBufferBackedInputStream(ByteBuffer buf) {
+ buffer = buf;
}
- private DataInputByteBuffer(DataInputByteBuffer.Buffer buffers) {
- super(buffers);
- this.buffers = buffers;
+ @Override
+ public int read() throws IOException {
+ return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
}
- public void reset(ByteBuffer... input) {
- this.buffers.reset(input);
- }
-
- public ByteBuffer[] getData() {
- return this.buffers.getData();
- }
-
- public int getPosition() {
- return this.buffers.getPosition();
- }
-
- public int getLength() {
- return this.buffers.getLength();
- }
-
- private static class Buffer extends InputStream {
- private final byte[] scratch;
- ByteBuffer[] buffers;
- int bidx;
- int pos;
- int length;
-
- private Buffer() {
- this.scratch = new byte[1];
- this.buffers = new ByteBuffer[0];
- }
-
- public int read() {
- return -1 == this.read(this.scratch, 0, 1) ? -1 : this.scratch[0] & 255;
+ @Override
+ public int read(byte[] bytes, int off, int len) throws IOException {
+ if (!buffer.hasRemaining()) {
+ return -1;
}
- public int read(byte[] b, int off, int len) {
- if (this.bidx >= this.buffers.length) {
- return -1;
- } else {
- int cur = 0;
-
- do {
- int rem = Math.min(len, this.buffers[this.bidx].remaining());
- this.buffers[this.bidx].get(b, off, rem);
- cur += rem;
- off += rem;
- len -= rem;
- } while (len > 0 && ++this.bidx < this.buffers.length);
-
- this.pos += cur;
- return cur;
- }
- }
-
- public void reset(ByteBuffer[] buffers) {
- this.bidx = this.pos = this.length = 0;
- this.buffers = buffers;
- ByteBuffer[] arr$ = buffers;
- int len$ = buffers.length;
-
- for (int i$ = 0; i$ < len$; ++i$) {
- ByteBuffer b = arr$[i$];
- this.length += b.remaining();
- }
-
- }
-
- public int getPosition() {
- return this.pos;
- }
-
- public int getLength() {
- return this.length;
- }
-
- public ByteBuffer[] getData() {
- return this.buffers;
- }
+ len = Math.min(len, buffer.remaining());
+ buffer.get(bytes, off, len);
+ return len;
}
}
}
[22/50] [abbrv] kylin git commit: KYLIN-2349 Serialize BitmapCounter
with peekLength
Posted by li...@apache.org.
KYLIN-2349 Serialize BitmapCounter with peekLength
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa574462
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa574462
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa574462
Branch: refs/heads/master-hbase1.x
Commit: aa574462aad0620fe10343613dc4c8d761c0d6a0
Parents: 746e380
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 31 15:41:07 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:08:15 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 53 ++++++++++++++------
1 file changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa574462/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a7f277e..aeb14ba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -34,6 +34,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
public class BitmapCounter implements Comparable<BitmapCounter> {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ private final int VERSION = 2;
public BitmapCounter() {
}
@@ -102,15 +103,51 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
bitmap.serialize(dos);
dos.close();
ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+ out.putInt(VERSION);
+ out.putInt(bos.size() + 4 + 4);
out.put(bb);
}
public void readRegisters(ByteBuffer in) throws IOException {
+ int mark = in.position();
+ int version = in.getInt();
+
+ // keep forward compatibility
+ if (version == VERSION) {
+ @SuppressWarnings("unused")
+ int size = in.getInt();
+ } else {
+ in.position(mark);
+ }
+
try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
bitmap.deserialize(is);
}
}
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+ int version = in.getInt();
+
+ // keep forward compatibility
+ if (version == VERSION) {
+ len = in.getInt() ;
+ } else {
+ in.position(mark);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ bitmap.deserialize(is);
+ len = in.position() - mark;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ in.position(mark);
+ return len;
+ }
+
@Override
public String toString() {
long count = getCount();
@@ -169,22 +206,6 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return -1;
}
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
-
- MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
- try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
- bitmap.deserialize(is);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
-
- len = in.position() - mark;
- in.position(mark);
- return len;
- }
-
private class ByteBufferBackedInputStream extends InputStream {
private final ByteBuffer buffer;
[16/50] [abbrv] kylin git commit: minor, skip JarScan in DebugTomcat
Posted by li...@apache.org.
minor, skip JarScan in DebugTomcat
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b85f710
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b85f710
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b85f710
Branch: refs/heads/master-hbase1.x
Commit: 1b85f710d91c7f7f5a81790d665a1ac617c7a591
Parents: fbacb71
Author: Li Yang <li...@apache.org>
Authored: Fri Jan 6 16:28:10 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 6 16:28:20 2017 +0800
----------------------------------------------------------------------
server/src/main/java/org/apache/kylin/rest/DebugTomcat.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b85f710/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 4f49402..2cac82e 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -25,6 +25,8 @@ import org.apache.catalina.startup.Tomcat;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Shell;
import org.apache.kylin.common.KylinConfig;
+import org.apache.tomcat.JarScanFilter;
+import org.apache.tomcat.JarScanType;
import org.apache.tomcat.util.descriptor.web.ErrorPage;
import java.io.File;
@@ -127,6 +129,12 @@ public class DebugTomcat {
notFound.setLocation("/index.html");
webContext.addErrorPage(notFound);
webContext.addWelcomeFile("index.html");
+ webContext.getJarScanner().setJarScanFilter(new JarScanFilter() {
+ @Override
+ public boolean check(JarScanType arg0, String arg1) {
+ return false;
+ }
+ });
// tomcat start
tomcat.start();
[36/50] [abbrv] kylin git commit: KYLIN-2331 use
org.apache.htrace.Trace to find htrace-core.jar
Posted by li...@apache.org.
KYLIN-2331 use org.apache.htrace.Trace to find htrace-core.jar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a69efa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a69efa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a69efa
Branch: refs/heads/master-hbase1.x
Commit: b9a69efaea27d915ae44f75db436b6c9b6b7a4ec
Parents: e864cd3
Author: shaofengshi <sh...@apache.org>
Authored: Tue Jan 3 13:36:49 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Jan 9 16:58:11 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a69efa/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index c5d47e7..9431468 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -59,10 +59,12 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
StringBuilder jars = new StringBuilder();
StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration"));
+ StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace")); // htrace-core.jar
+ StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar
StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar
StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar
StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar
+ StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar
StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars());
sparkExecutable.setJars(jars.toString());
[27/50] [abbrv] kylin git commit: KYLIN-2367 fix select * where ...
Posted by li...@apache.org.
KYLIN-2367 fix select * where ...
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f8cdeb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f8cdeb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f8cdeb5
Branch: refs/heads/master-hbase1.x
Commit: 6f8cdeb55cb4b41a63871aed2b3f61c7eddd107d
Parents: 68ff75d
Author: Li Yang <li...@apache.org>
Authored: Mon Jan 9 14:41:05 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Jan 9 14:41:05 2017 +0800
----------------------------------------------------------------------
.../org/apache/calcite/sql2rel/SqlToRelConverter.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f8cdeb5/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 7da3040..e0782ce 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -677,7 +677,8 @@ public class SqlToRelConverter {
*
* LogicalSort (optional)
* |- LogicalProject
- * |- OLAPTableScan
+ * |- LogicalFilter (optional)
+ * |- OLAPTableScan
*/
LogicalProject rootPrj = null;
LogicalSort rootSort = null;
@@ -690,11 +691,14 @@ public class SqlToRelConverter {
return root;
}
- if (!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+ RelNode input = rootPrj.getInput();
+ if (!(//
+ input.getClass().getSimpleName().equals("OLAPTableScan")//
+ || (input.getClass().getSimpleName().equals("LogicalFilter") && input.getInput(0).getClass().getSimpleName().equals("OLAPTableScan"))//
+ ))
return root;
- RelNode scan = rootPrj.getInput();
- if (rootPrj.getRowType().getFieldCount() < scan.getRowType().getFieldCount())
+ if (rootPrj.getRowType().getFieldCount() < input.getRowType().getFieldCount())
return root;
RelDataType inType = rootPrj.getRowType();
@@ -713,7 +717,7 @@ public class SqlToRelConverter {
}
RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder);
- rootPrj = LogicalProject.create(scan, projExp, projRowType);
+ rootPrj = LogicalProject.create(input, projExp, projRowType);
if (rootSort != null) {
rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
}
[02/50] [abbrv] kylin git commit: KYLIN-2359 Update job build step
name
Posted by li...@apache.org.
KYLIN-2359 Update job build step name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c985742
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c985742
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c985742
Branch: refs/heads/master-hbase1.x
Commit: 9c985742584a6bf15053d5482d13e1e01f76894d
Parents: 809b783
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 12:19:42 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 12:19:42 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/constant/ExecutableConstants.java | 7 ++++---
.../org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java | 4 ++--
.../org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java | 4 ++--
3 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index 4ae3add..11c7455 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -38,9 +38,9 @@ public final class ExecutableConstants {
public static final String STEP_NAME_CREATE_FLAT_HIVE_TABLE = "Create Intermediate Flat Hive Table";
public static final String STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP = "Materialize Hive View in Lookup Tables";
public static final String STEP_NAME_FACT_DISTINCT_COLUMNS = "Extract Fact Table Distinct Columns";
- public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid Data";
- public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube";
- public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension Cuboid Data";
+ public static final String STEP_NAME_BUILD_BASE_CUBOID = "Build Base Cuboid";
+ public static final String STEP_NAME_BUILD_IN_MEM_CUBE = "Build Cube In-Mem";
+ public static final String STEP_NAME_BUILD_N_D_CUBOID = "Build N-Dimension Cuboid";
public static final String STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION = "Calculate HTable Region Splits";
public static final String STEP_NAME_CREATE_HBASE_TABLE = "Create HTable";
public static final String STEP_NAME_CONVERT_CUBOID_TO_HFILE = "Convert Cuboid Data to HFile";
@@ -53,6 +53,7 @@ public final class ExecutableConstants {
public static final String STEP_NAME_HIVE_CLEANUP = "Hive Cleanup";
public static final String STEP_NAME_KAFKA_CLEANUP = "Kafka Intermediate File Cleanup";
public static final String STEP_NAME_GARBAGE_COLLECTION = "Garbage Collection";
+ public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage Collection on HBase";
public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table";
public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 20540a6..9229a75 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -43,13 +43,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
public BatchCubingJobBuilder2(CubeSegment newSegment, String submitter) {
super(newSegment, submitter);
this.inputSide = MRUtil.getBatchCubingInputSide(seg);
- this.outputSide = MRUtil.getBatchCubingOutputSide2((CubeSegment) seg);
+ this.outputSide = MRUtil.getBatchCubingOutputSide2(seg);
}
public CubingJob build() {
logger.info("MR_V2 new job to BUILD segment " + seg);
- final CubingJob result = CubingJob.createBuildJob((CubeSegment) seg, submitter, config);
+ final CubingJob result = CubingJob.createBuildJob(seg, submitter, config);
final String jobId = result.getId();
final String cuboidRootPath = getCuboidRootPath(jobId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9c985742/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
index 7c2b3fd..6f69e8c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMRSteps.java
@@ -162,7 +162,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
public MergeGCStep createMergeGCStep() {
MergeGCStep result = new MergeGCStep();
- result.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+ result.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HBASE);
result.setOldHTables(getMergingHTables());
return result;
}
@@ -218,7 +218,7 @@ public class HBaseMRSteps extends JobBuilderSupport {
toDeletePaths.add(getFactDistinctColumnsPath(jobId));
HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep();
- step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
+ step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HBASE);
step.setDeletePaths(toDeletePaths);
step.setJobId(jobId);
[10/50] [abbrv] kylin git commit: KYLIN-2351 Support using
non-default HDFS path as kylin.env.hdfs-working-dir
Posted by li...@apache.org.
KYLIN-2351 Support using non-default HDFS path as kylin.env.hdfs-working-dir
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/399f0c45
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/399f0c45
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/399f0c45
Branch: refs/heads/master-hbase1.x
Commit: 399f0c45108115c627cbf5f452ec33ff94167247
Parents: 503d232
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 4 18:24:48 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Jan 6 06:45:37 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 9 +++++++
.../common/util/LocalFileMetadataTestCase.java | 4 ++-
.../kylin/dict/AppendTrieDictionaryTest.java | 2 +-
.../org/apache/kylin/engine/mr/HadoopUtil.java | 28 +++++++++++++++++++-
.../engine/mr/common/AbstractHadoopJob.java | 4 +--
.../engine/mr/steps/CreateDictionaryJob.java | 2 +-
.../mr/steps/FactDistinctColumnsReducer.java | 10 ++++---
.../engine/mr/steps/MergeStatisticsStep.java | 2 +-
.../steps/RowKeyDistributionCheckerMapper.java | 3 ++-
.../engine/mr/steps/SaveStatisticsStep.java | 2 +-
.../kylin/engine/mr/SortedColumnReaderTest.java | 20 +++++++++-----
.../kylin/provision/BuildCubeWithEngine.java | 3 +--
.../apache/kylin/query/ITMassInQueryTest.java | 3 +--
.../kylin/storage/hbase/ITHdfsOpsTest.java | 4 +--
.../apache/kylin/source/hive/HiveMRInput.java | 2 +-
.../apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 11 +++-----
.../storage/hbase/steps/DeprecatedGCStep.java | 3 +--
.../steps/HDFSPathGarbageCollectionStep.java | 2 +-
.../hbase/steps/SequenceFileCuboidWriter.java | 2 +-
.../storage/hbase/util/CubeMigrationCLI.java | 3 ++-
.../storage/hbase/util/StorageCleanupJob.java | 3 ++-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 3 ++-
.../apache/kylin/tool/StorageCleanupJob.java | 4 +--
24 files changed, 87 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3e6384f..88c4dbc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -171,6 +171,15 @@ abstract public class KylinConfigBase implements Serializable {
if (!root.endsWith("/")) {
root += "/";
}
+
+ // make sure path qualified
+ if (!root.contains("://")) {
+ if (!root.startsWith("/"))
+ root = "hdfs:///" + root;
+ else
+ root = "hdfs://" + root;
+ }
+
return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
index 26e1e86..4a24941 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/LocalFileMetadataTestCase.java
@@ -57,7 +57,9 @@ public class LocalFileMetadataTestCase extends AbstractKylinTestCase {
if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null)
System.setProperty(KylinConfig.KYLIN_CONF, tempTestMetadataUrl);
- KylinConfig.getInstanceFromEnv().setMetadataUrl(tempTestMetadataUrl);
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setMetadataUrl(tempTestMetadataUrl);
+ config.setProperty("kylin.env.hdfs-working-dir", "file:///tmp/kylin");
}
public static void cleanAfterClass() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index f84f08f..921925c 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -54,7 +54,7 @@ import org.junit.Test;
*/
public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
- public static final String BASE_DIR = "/tmp/kylin_append_dict";
+ public static final String BASE_DIR = "file:///tmp/kylin_append_dict";
public static final String RESOURCE_DIR = "/dict/append_dict_test";
@Before
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..3119c1e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Writable;
+import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,8 +67,33 @@ public class HadoopUtil {
return conf;
}
+ public static FileSystem getWorkingFileSystem() throws IOException {
+ return getFileSystem(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ }
+
+ public static FileSystem getWorkingFileSystem(Configuration conf) throws IOException {
+ Path workingPath = new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory());
+ return getFileSystem(workingPath, conf);
+ }
+
public static FileSystem getFileSystem(String path) throws IOException {
- return FileSystem.get(makeURI(path), getCurrentConfiguration());
+ return getFileSystem(new Path(makeURI(path)));
+ }
+
+ public static FileSystem getFileSystem(Path path) throws IOException {
+ Configuration conf = getCurrentConfiguration();
+ return getFileSystem(path, conf);
+ }
+
+ public static FileSystem getFileSystem(Path path, Configuration conf) {
+ if (StringUtils.isBlank(path.toUri().getScheme()))
+ throw new IllegalArgumentException("Path must be qualified: " + path);
+
+ try {
+ return path.getFileSystem(conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
public static URI makeURI(String filePath) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index e4e5e82..1540acd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -283,7 +283,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
try {
Configuration jobConf = job.getConfiguration();
FileSystem localfs = FileSystem.getLocal(jobConf);
- FileSystem hdfs = FileSystem.get(jobConf);
+ FileSystem hdfs = HadoopUtil.getWorkingFileSystem(jobConf);
StringBuilder jarList = new StringBuilder();
StringBuilder fileList = new StringBuilder();
@@ -407,7 +407,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
inp = inp.trim();
if (inp.endsWith("/*")) {
inp = inp.substring(0, inp.length() - 2);
- FileSystem fs = FileSystem.get(job.getConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(job.getConfiguration());
Path path = new Path(inp);
if (!exists(fs, path)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 8b9697e..b27d722 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -64,7 +64,7 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(dictFile) == false)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5d42797..3c7f951 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -43,6 +43,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IDictionaryBuilder;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -213,7 +214,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
- final FileSystem fs = FileSystem.get(conf);
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
final Path colDir = new Path(outputPath, col.getIdentity());
final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
@@ -269,7 +270,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
final Configuration conf = context.getConfiguration();
- final FileSystem fs = FileSystem.get(conf);
+ final FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
final Path outputFile = new Path(outputPath, outputFileName);
if (!fs.exists(outputPath)) {
@@ -315,8 +316,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private void writeMapperAndCuboidStatistics(Context context) throws IOException {
Configuration conf = context.getConfiguration();
- FileSystem fs = FileSystem.get(conf);
- FSDataOutputStream out = fs.create(new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME));
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
+ Path path = new Path(statisticsOutput, BatchConstants.CFG_STATISTICS_CUBE_ESTIMATION_FILENAME);
+ FSDataOutputStream out = fs.create(path);
try {
String msg;
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 811fc24..54666d0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -123,7 +123,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
averageSamplingPercentage = averageSamplingPercentage / CubingExecutableUtil.getMergingSegmentIds(this.getParams()).size();
CubeStatsWriter.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage);
Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- FileSystem fs = statisticsFilePath.getFileSystem(conf);
+ FileSystem fs = HadoopUtil.getFileSystem(statisticsFilePath, conf);
FSDataInputStream is = fs.open(statisticsFilePath);
try {
// put the statistics to metadata store
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
index fca91a6..ee8da6b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/RowKeyDistributionCheckerMapper.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.KylinMapper;
/**
@@ -89,7 +90,7 @@ public class RowKeyDistributionCheckerMapper extends KylinMapper<Text, Text, Tex
List<byte[]> rowkeyList = new ArrayList<byte[]>();
SequenceFile.Reader reader = null;
try {
- reader = new SequenceFile.Reader(path.getFileSystem(conf), path, conf);
+ reader = new SequenceFile.Reader(HadoopUtil.getFileSystem(path, conf), path, conf);
Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
while (reader.next(key, value)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..6120270 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -59,8 +59,8 @@ public class SaveStatisticsStep extends AbstractExecutable {
ResourceStore rs = ResourceStore.getStore(kylinConf);
try {
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
Path statisticsFilePath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
- FileSystem fs = FileSystem.get(HadoopUtil.getCurrentConfiguration());
if (!fs.exists(statisticsFilePath))
throw new IOException("File " + statisticsFilePath + " does not exists");
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
index 3c4195f..be440f6 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/SortedColumnReaderTest.java
@@ -48,10 +48,9 @@ public class SortedColumnReaderTest {
@Test
public void testReadStringMultiFile() throws Exception {
String dirPath = "src/test/resources/multi_file_str";
- StringBytesConverter converter = new StringBytesConverter();
ArrayList<String> correctAnswer = readAllFiles(dirPath);
Collections.sort(correctAnswer, new ByteComparator<String>(new StringBytesConverter()));
- SortedColumnDFSFile column = new SortedColumnDFSFile(dirPath + "/", DataType.getType("varchar"));
+ SortedColumnDFSFile column = new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("varchar"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
@@ -124,7 +123,7 @@ public class SortedColumnReaderTest {
}
}
});
- SortedColumnDFSFile column = new SortedColumnDFSFile(dirPath + "/", DataType.getType("long"));
+ SortedColumnDFSFile column = new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("long"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
@@ -142,7 +141,7 @@ public class SortedColumnReaderTest {
public void testEmptyDir() throws Exception {
String dirPath = "src/test/resources/empty_dir";
new File(dirPath).mkdirs();
- SortedColumnDFSFile column = new SortedColumnDFSFile(dirPath + "/", DataType.getType("varchar"));
+ SortedColumnDFSFile column = new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("varchar"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
@@ -159,7 +158,7 @@ public class SortedColumnReaderTest {
final BytesConverter<String> converter = new StringBytesConverter();
Collections.sort(correctAnswer, new ByteComparator<String>(new StringBytesConverter()));
System.out.println("correct answer:" + correctAnswer);
- SortedColumnDFSFile column = new SortedColumnDFSFile(dirPath + "/", DataType.getType("varchar"));
+ SortedColumnDFSFile column = new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("varchar"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
@@ -230,7 +229,7 @@ public class SortedColumnReaderTest {
}
}
});
- SortedColumnDFSFile column = new SortedColumnDFSFile(dirPath + "/", DataType.getType("double"));
+ SortedColumnDFSFile column = new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("double"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
@@ -300,4 +299,13 @@ public class SortedColumnReaderTest {
}
return result;
}
+
+ private String qualify(String path) {
+ String absolutePath = new File(path).getAbsolutePath();
+ if (absolutePath.startsWith("/"))
+ return "file://" + absolutePath;
+ else
+ return "file:///" + absolutePath;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 1c7c64d..62978db 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -120,8 +120,7 @@ public class BuildCubeWithEngine {
try {
//check hdfs permission
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
Path coprocessorDir = new Path(hdfsWorkingDirectory);
boolean success = fileSystem.mkdirs(coprocessorDir);
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 5835baf..18d79ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -68,8 +68,7 @@ public class ITMassInQueryTest extends KylinTestBase {
ITKylinQueryTest.joinType = "left";
ITKylinQueryTest.setupAll();
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- fileSystem = FileSystem.get(hconf);
+ fileSystem = HadoopUtil.getWorkingFileSystem();
int sellerCount = 200;
Random r = new Random();
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 499a456..786d7d1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -42,9 +42,7 @@ public class ITHdfsOpsTest extends HBaseMetadataTestCase {
this.createTestMetadata();
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
-
- fileSystem = FileSystem.get(hconf);
+ fileSystem = HadoopUtil.getWorkingFileSystem();
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..0f351b0 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -379,7 +379,7 @@ public class HiveMRInput implements IMRInput {
private void rmdirOnHDFS(String path) throws IOException {
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..2f4fded 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -223,7 +223,7 @@ public class KafkaMRInput implements IMRInput {
private void rmdirOnHDFS(String path) throws IOException {
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1d19983..2a12984 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -27,7 +27,6 @@ import java.util.NavigableSet;
import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -49,6 +48,7 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -201,8 +201,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] value = r.getValue(B_FAMILY, B_COLUMN);
if (value.length == 0) {
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
return fileSystem.open(redirectPath);
} else {
@@ -292,8 +291,7 @@ public class HBaseResourceStore extends ResourceStore {
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
if (fileSystem.exists(redirectPath)) {
fileSystem.delete(redirectPath, true);
@@ -339,8 +337,7 @@ public class HBaseResourceStore extends ResourceStore {
private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
if (fileSystem.exists(redirectPath)) {
fileSystem.delete(redirectPath, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 46a828e..fbe64d9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -132,8 +132,7 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldHdfsPaths = this.getOldHdsfPaths();
if (oldHdfsPaths != null && oldHdfsPaths.size() > 0) {
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
for (String path : oldHdfsPaths) {
if (path.endsWith("*"))
path = path.substring(0, path.length() - 1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
index fbfd582..89baf95 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HDFSPathGarbageCollectionStep.java
@@ -59,7 +59,7 @@ public class HDFSPathGarbageCollectionStep extends AbstractExecutable {
try {
config = new JobEngineConfig(context.getConfig());
List<String> toDeletePaths = getDeletePaths();
- dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HadoopUtil.getCurrentConfiguration()));
+ dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem());
if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) {
dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
index 8f2fc80..1d66d3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/SequenceFileCuboidWriter.java
@@ -55,7 +55,7 @@ public class SequenceFileCuboidWriter extends KVGTRecordWriter {
JobBuilderSupport jobBuilderSupport = new JobBuilderSupport(cubeSegment, "SYSTEM");
String cuboidRoot = jobBuilderSupport.getCuboidRootPath(cubeSegment);
Path cuboidPath = new Path(cuboidRoot);
- FileSystem fs = HadoopUtil.getFileSystem(cuboidRoot);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
try {
if (fs.exists(cuboidPath)) {
fs.delete(cuboidPath, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..31864f6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -55,6 +55,7 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -136,7 +137,7 @@ public class CubeMigrationCLI {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
hbaseAdmin = new HBaseAdmin(conf);
- hdfsFS = FileSystem.get(new Configuration());
+ hdfsFS = HadoopUtil.getWorkingFileSystem();
operations = new ArrayList<Opt>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 9fe5a23..d1a74ad 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -50,6 +50,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -180,7 +181,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 40306c9..5269195 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -55,6 +55,7 @@ import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -134,7 +135,7 @@ public class CubeMigrationCLI {
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
hbaseAdmin = new HBaseAdmin(conf);
- hdfsFS = FileSystem.get(new Configuration());
+ hdfsFS = HadoopUtil.getWorkingFileSystem();
operations = new ArrayList<Opt>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/399f0c45/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..b3e2ec0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -185,7 +185,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = HadoopUtil.getWorkingFileSystem(conf);
List<String> allHdfsPathsNeedToBeDeleted = new ArrayList<String>();
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
@@ -342,7 +342,7 @@ public class StorageCleanupJob extends AbstractApplication {
if (segmentId2JobId.containsKey(segmentId)) {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
Path externalDataPath = new Path(path);
- FileSystem fs = FileSystem.get(externalDataPath.toUri(), HadoopUtil.getCurrentConfiguration());
+ FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
[07/50] [abbrv] kylin git commit: KYLIN-2360 fix potential bugs
reported by Sonar
Posted by li...@apache.org.
KYLIN-2360 fix potential bugs reported by Sonar
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c64f55b2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c64f55b2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c64f55b2
Branch: refs/heads/master-hbase1.x
Commit: c64f55b2e34089d10c8a55301b62f96aa597d8f1
Parents: 5683954
Author: Billy Liu <bi...@apache.org>
Authored: Thu Jan 5 17:41:06 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 5 17:41:06 2017 +0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableWindow.java | 48 --------------------
.../apache/kylin/common/KylinConfigBase.java | 11 +++--
.../common/util/MemoryBudgetController.java | 4 +-
.../apache/kylin/common/util/ThreadUtil.java | 47 -------------------
.../org/apache/kylin/common/util/TimeUtil.java | 2 +-
.../apache/kylin/common/util/ZipFileUtils.java | 5 +-
.../kylin/cube/cli/DumpDictionaryCLI.java | 6 ++-
.../cube/inmemcubing/DoggedCubeBuilder.java | 3 ++
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +
.../kylin/cube/inmemcubing/MemDiskStore.java | 1 +
.../kylin/cube/model/AggregationGroup.java | 3 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 3 +-
.../apache/kylin/cube/model/DictionaryDesc.java | 4 +-
.../apache/kylin/cube/model/DimensionDesc.java | 3 +-
.../kylin/cube/model/HBaseMappingDesc.java | 3 +-
.../org/apache/kylin/cube/model/RowKeyDesc.java | 3 +-
.../kylin/cube/model/v1_4_0/CubeDesc.java | 3 +-
.../kylin/gridtable/GTAggregateScanner.java | 4 +-
.../org/apache/kylin/dict/CachedTreeMap.java | 16 +++----
.../org/apache/kylin/dict/TrieDictionary.java | 2 +-
.../apache/kylin/dict/TrieDictionaryForest.java | 2 +-
.../apache/kylin/dimension/BooleanDimEnc.java | 2 +-
.../kylin/measure/MeasureTypeFactory.java | 3 +-
.../kylin/metadata/model/FunctionDesc.java | 8 ++--
.../kylin/metadata/model/JoinTableDesc.java | 4 +-
.../kylin/metadata/model/MeasureDesc.java | 3 +-
.../metadata/model/ModelDimensionDesc.java | 9 ++--
.../kylin/metadata/model/PartitionDesc.java | 4 +-
.../apache/kylin/metadata/model/TableDesc.java | 2 +-
.../kylin/metadata/model/TableExtDesc.java | 3 +-
.../apache/kylin/metadata/model/TableRef.java | 3 +-
.../metadata/project/RealizationEntry.java | 4 +-
.../org/apache/kylin/metadata/tuple/Tuple.java | 2 +-
.../apache/kylin/metadata/tuple/TupleInfo.java | 10 ++--
.../org/apache/kylin/source/ReadableTable.java | 3 +-
.../kylin/source/datagen/ColumnGenerator.java | 2 +-
.../kylin/engine/mr/DFSFileTableReader.java | 4 +-
.../engine/mr/common/HadoopStatusGetter.java | 2 +
.../engine/mr/common/MapReduceExecutable.java | 3 +-
.../mr/steps/MapContextGTRecordWriter.java | 1 +
.../engine/mr/steps/SelfDefineSortableKey.java | 2 +-
.../apache/kylin/engine/spark/SparkCubing.java | 4 ++
.../engine/spark/cube/BufferedCuboidWriter.java | 3 ++
server-base/pom.xml | 7 ---
.../kylin/rest/request/PrepareSqlRequest.java | 3 +-
.../kylin/rest/service/BadQueryDetector.java | 5 +-
.../apache/kylin/rest/service/CubeService.java | 4 ++
.../kylin/rest/service/DiagnosisService.java | 6 ++-
.../cardinality/ColumnCardinalityMapper.java | 3 +-
source-kafka/pom.xml | 1 -
.../kylin/source/kafka/config/BrokerConfig.java | 4 +-
.../common/coprocessor/AggregationCache.java | 4 +-
.../hbase/cube/v2/ExpectedSizeIterator.java | 2 +
.../storage/hbase/util/HbaseStreamingInput.java | 2 +
.../kylin/storage/hbase/util/PingHBaseCLI.java | 1 +
.../hbase/util/ZookeeperDistributedJobLock.java | 2 +
.../org/apache/kylin/tool/CubeMetaIngester.java | 3 +-
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 1 +
.../apache/kylin/tool/HBaseUsageExtractor.java | 2 +-
59 files changed, 137 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
index 203ce02..547210c 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -113,54 +113,6 @@ public class EnumerableWindow extends Window implements EnumerableRel {
}
}
- private void sampleOfTheGeneratedWindowedAggregate() {
- // Here's overview of the generated code
- // For each list of rows that have the same partitioning key, evaluate
- // all of the windowed aggregate functions.
-
- // builder
- Iterator<Integer[]> iterator = null;
-
- // builder3
- Integer[] rows = iterator.next();
-
- int prevStart = -1;
- int prevEnd = -1;
-
- for (int i = 0; i < rows.length; i++) {
- // builder4
- Integer row = rows[i];
-
- int start = 0;
- int end = 100;
- if (start != prevStart || end != prevEnd) {
- // builder5
- int actualStart = 0;
- if (start != prevStart || end < prevEnd) {
- // builder6
- // recompute
- actualStart = start;
- // implementReset
- } else { // must be start == prevStart && end > prevEnd
- actualStart = prevEnd + 1;
- }
- prevStart = start;
- prevEnd = end;
-
- if (start != -1) {
- for (int j = actualStart; j <= end; j++) {
- // builder7
- // implementAdd
- }
- }
- // implementResult
- // list.add(new Xxx(row.deptno, row.empid, sum, count));
- }
- }
- // multiMap.clear(); // allows gc
- // source = Linq4j.asEnumerable(list);
- }
-
public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
final JavaTypeFactory typeFactory = implementor.getTypeFactory();
final EnumerableRel child = (EnumerableRel) getInput();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bb8880b..3e6384f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -533,10 +533,13 @@ abstract public class KylinConfigBase implements Serializable {
File home = new File(homePath);
SortedSet<String> files = Sets.newTreeSet();
if (home.exists() && home.isDirectory()) {
- for (File file : home.listFiles()) {
- final Matcher matcher = pattern.matcher(file.getName());
- if (matcher.matches()) {
- files.add(file.getAbsolutePath());
+ File[] listFiles = home.listFiles();
+ if (listFiles != null) {
+ for (File file : listFiles) {
+ final Matcher matcher = pattern.matcher(file.getName());
+ if (matcher.matches()) {
+ files.add(file.getAbsolutePath());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 920f8e6..ade929c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -58,7 +58,7 @@ public class MemoryBudgetController {
public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
public static final int ONE_MB = 1024 * 1024;
- public static final long ONE_GB = 1024 * 1024 * 1024;
+ public static final long ONE_GB = 1024 * 1024 * 1024L;
private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
@@ -119,6 +119,7 @@ public class MemoryBudgetController {
try {
lock.wait();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new NotEnoughBudgetException(e);
}
}
@@ -256,6 +257,7 @@ public class MemoryBudgetController {
lastMB = thisMB;
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
logger.error("", e);
return getSystemAvailMB();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
deleted file mode 100644
index aebbc57..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.util.concurrent.Future;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- */
-public class ThreadUtil {
- @SuppressWarnings("unused")
- public static void main(String[] args) {
- ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable"));
-
- for (int i = 0; i < Integer.MAX_VALUE; ++i) {
- System.out.println("index: " + i);
- Future<?> future = pool.submit(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
index d3d4283..7a24c10 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java
@@ -26,7 +26,7 @@ import java.util.TimeZone;
public class TimeUtil {
private static TimeZone gmt = TimeZone.getTimeZone("GMT");
- private static long ONE_MINUTE_TS = 60 * 1000;
+ private static long ONE_MINUTE_TS = 60 * 1000L;
private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS;
private static long ONE_DAY_TS = 24 * ONE_HOUR_TS;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
index 7ee770d..a2326ee 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
@@ -77,7 +77,10 @@ public class ZipFileUtils {
}
private static void compressDirectoryToZipfile(String rootDir, String sourceDir, ZipOutputStream out) throws IOException {
- for (File sourceFile : new File(sourceDir).listFiles()) {
+ File[] files = new File(sourceDir).listFiles();
+ if (files == null)
+ return;
+ for (File sourceFile : files) {
if (sourceFile.isDirectory()) {
compressDirectoryToZipfile(rootDir, sourceDir + normDir(sourceFile.getName()), out);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
index 24daddf..729a6da 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
@@ -38,7 +38,11 @@ public class DumpDictionaryCLI {
public static void dump(File f) throws IOException {
if (f.isDirectory()) {
- for (File c : f.listFiles())
+ File[] files = f.listFiles();
+ if (files == null) {
+ return;
+ }
+ for (File c : files)
dump(c);
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 413c448..a9211da 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -180,6 +180,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
try {
split.join();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
errors.add(e);
}
if (split.exception != null)
@@ -221,6 +222,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
return false;
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
@@ -239,6 +241,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
last.join();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 66a444e..85a694f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -205,6 +205,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
for (Thread t : threads)
t.join();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException("interrupted while waiting task and output complete", e);
}
}
@@ -531,6 +532,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
try {
currentObject = input.take();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
return currentObject != null && currentObject.size() > 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
index 3cfaed7..81403ab 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/MemDiskStore.java
@@ -603,6 +603,7 @@ public class MemDiskStore implements IGTStore, Closeable {
} catch (NullPointerException npe) {
// that's fine, async flusher may not present
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
logger.warn("async join interrupted", e);
}
synchronized (lock) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index 9bd082f..1d4d496 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.model;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -35,7 +36,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class AggregationGroup {
+public class AggregationGroup implements Serializable{
public static class HierarchyMask {
public long fullMask; // 00000111
public long[] allMasks; // 00000100,00000110,00000111
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4686a15..556cc72 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -825,7 +825,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
if (infoList == null) {
- hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
+ infoList = new ArrayList<DeriveInfo>();
+ hostToDerivedMap.put(hostColArray, infoList);
}
// Merged duplicated derived column
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
index f471f9f..fb7053e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java
@@ -26,8 +26,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DictionaryDesc {
+public class DictionaryDesc implements Serializable{
@JsonProperty("column")
private String column;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index cd75228..29fd6c3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.model;
+import java.io.Serializable;
import java.util.Arrays;
import org.apache.commons.lang.NotImplementedException;
@@ -35,7 +36,7 @@ import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc {
+public class DimensionDesc implements Serializable {
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index 2ef1e17..84d87de 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.model;
+import java.io.Serializable;
import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
@@ -33,7 +34,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseMappingDesc {
+public class HBaseMappingDesc implements Serializable{
@JsonProperty("column_family")
private HBaseColumnFamilyDesc[] columnFamily;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index f1a403d..1bd06d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.model;
+import java.io.Serializable;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -35,7 +36,7 @@ import com.google.common.base.Objects;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyDesc {
+public class RowKeyDesc implements Serializable{
@JsonProperty("rowkey_columns")
private RowKeyColDesc[] rowkeyColumns;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
index bd73bc8..4019e4f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
@@ -630,7 +630,8 @@ public class CubeDesc extends RootPersistentEntity {
Map<String, TblColRef> cols = columnMap.get(ref.getTable());
if (cols == null) {
- columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>());
+ cols = new HashMap<String, TblColRef>();
+ columnMap.put(ref.getTable(), cols);
}
cols.put(ref.getName(), ref);
return ref;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 9bbcd10..55c04c6 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -99,7 +99,7 @@ public class GTAggregateScanner implements IGTScanner {
public static long estimateSizeOf(MeasureAggregator[] aggrs) {
// size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc..
// Memory alignment to 8 bytes
- long est = (aggrs.length + 1) / 2 * 8 + 4 + (4 /* extra */);
+ long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */);
for (MeasureAggregator aggr : aggrs) {
if (aggr != null)
est += aggr.getMemBytesEstimate();
@@ -110,7 +110,7 @@ public class GTAggregateScanner implements IGTScanner {
public static long estimateSizeOf(byte[] bytes) {
// AggregationCacheMemSizeTest reports 20 for byte[10] and 20 again for byte[16]
// Memory alignment to 8 bytes
- return (bytes.length + 7) / 8 * 8 + 4 + (4 /* extra */);
+ return (bytes.length + 7) / 8 * 8L + 4 + (4 /* extra */);
}
public void trackMemoryLevel(MemoryWaterLevel tracker) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 6acf764..3be4fd7 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -269,7 +269,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
}
public void commit(boolean keepAppend) throws IOException {
- assert this.keepAppend & !immutable : "Only support commit method with immutable false and keepAppend true";
+ assert this.keepAppend && !immutable : "Only support commit method with immutable false and keepAppend true";
Path newVersionDir = new Path(baseDir, VERSION_PREFIX + System.currentTimeMillis());
if (keepAppend) {
@@ -312,7 +312,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
}
String fileName = generateFileName(key);
Path filePath = new Path(fileName);
- try (FSDataOutputStream out = fs.create(filePath, true, BUFFER_SIZE, (short) 5, BUFFER_SIZE * 8)) {
+ try (FSDataOutputStream out = fs.create(filePath, true, BUFFER_SIZE, (short) 5, BUFFER_SIZE * 8L)) {
value.write(out);
} catch (Exception e) {
logger.error(String.format("write value into %s exception: %s", fileName, e), e);
@@ -350,7 +350,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
@Override
public V put(K key, V value) {
- assert keepAppend & !immutable : "Only support put method with immutable false and keepAppend true";
+ assert keepAppend && !immutable : "Only support put method with immutable false and keepAppend true";
super.put(key, null);
valueCache.put(key, value);
return null;
@@ -362,7 +362,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
try {
return valueCache.get((K) key);
} catch (ExecutionException e) {
- logger.error(String.format("get value with key %s exception: ", key, e), e);
+ logger.error(String.format("get value with key %s exception: %s", key, e), e);
return null;
}
} else {
@@ -372,7 +372,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
@Override
public V remove(Object key) {
- assert keepAppend & !immutable : "Only support remove method with immutable false keepAppend true";
+ assert keepAppend && !immutable : "Only support remove method with immutable false keepAppend true";
super.remove(key);
valueCache.invalidate(key);
return null;
@@ -421,21 +421,21 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
try {
return (V) valueCache.get(currentKey);
} catch (ExecutionException e) {
- logger.error(String.format("get value with key %s exception: ", currentKey, e), e);
+ logger.error(String.format("get value with key %s exception: %s", currentKey, e), e);
return null;
}
}
@Override
public void remove() {
- assert keepAppend & !immutable : "Only support remove method with immutable false and keepAppend true";
+ assert keepAppend && !immutable : "Only support remove method with immutable false and keepAppend true";
keyIterator.remove();
valueCache.invalidate(currentKey);
}
}
public FSDataOutputStream openIndexOutput() throws IOException {
- assert keepAppend & !immutable : "Only support write method with immutable false and keepAppend true";
+ assert keepAppend && !immutable : "Only support write method with immutable false and keepAppend true";
Path indexPath = new Path(getCurrentDir(), ".index");
return fs.create(indexPath, true, 8 * 1024 * 1024, (short) 5, 8 * 1024 * 1024 * 8);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 957207e..8e7f5dc 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -107,7 +107,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance();
this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
- this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
+ this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
enableCache();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 065c3df..69b4b9c 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -91,7 +91,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
return 1;
int maxOffset = accuOffset.get(accuOffset.size() - 1);
TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
- int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1);
+ int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
return sizeOfId;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index c3f4c11..08fbe51 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -63,7 +63,7 @@ public class BooleanDimEnc extends DimensionEncoding {
// ============================================================================
- private static int fixedLen = 1;
+ private int fixedLen = 1;
//no-arg constructor is required for Externalizable
public BooleanDimEnc() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 694459b..8e58858 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -142,7 +142,8 @@ abstract public class MeasureTypeFactory<T> {
registerUDAF(factory);
List<MeasureTypeFactory<?>> list = factories.get(funcName);
if (list == null)
- factories.put(funcName, list = Lists.newArrayListWithCapacity(2));
+ list = Lists.newArrayListWithCapacity(2);
+ factories.put(funcName, list);
list.add(factory);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 4d89e1a..184c290 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -95,7 +95,8 @@ public class FunctionDesc {
}
}
- parameter.setColRefs(colRefs);
+ if(parameter != null)
+ parameter.setColRefs(colRefs);
}
private void reInitMeasureType() {
@@ -265,10 +266,7 @@ public class FunctionDesc {
if (other.parameter != null)
return false;
} else {
- if (parameter == null) {
- if (other.parameter != null)
- return false;
- } else if (!parameter.equals(other.parameter))
+ if (!parameter.equals(other.parameter))
return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
index 5d0409a..513217e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java
@@ -25,8 +25,10 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class JoinTableDesc {
+public class JoinTableDesc implements Serializable{
@JsonProperty("table")
private String table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index 253b06b..6ddbbf4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -29,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class MeasureDesc {
+public class MeasureDesc implements Serializable{
@JsonProperty("name")
private String name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index 6460f71..bd3dfef 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
import java.util.List;
import org.apache.kylin.common.util.StringUtil;
@@ -28,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ModelDimensionDesc {
+public class ModelDimensionDesc implements Serializable{
@JsonProperty("table")
private String table;
@JsonProperty("columns")
@@ -58,8 +59,10 @@ public class ModelDimensionDesc {
if (model != null) {
table = model.findTable(table).getAlias();
- for (int i = 0; i < columns.length; i++) {
- columns[i] = model.findColumn(table, columns[i]).getName();
+ if (columns != null) {
+ for (int i = 0; i < columns.length; i++) {
+ columns[i] = model.findColumn(table, columns[i]).getName();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 9925990..bab3ec3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -27,10 +27,12 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class PartitionDesc {
+public class PartitionDesc implements Serializable{
public static enum PartitionType {
APPEND, //
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index e845da1..e4fc4c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -130,7 +130,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
this.name = splits[0];
}
} else {
- this.name = name;
+ this.name = null;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
index 97dd2c1..7ed3368 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -181,7 +182,7 @@ public class TableExtDesc extends RootPersistentEntity {
}
@JsonIgnoreProperties(ignoreUnknown = true)
- public static class ColumnStats implements Comparable<ColumnStats> {
+ public static class ColumnStats implements Comparable<ColumnStats>, Serializable {
@JsonBackReference
private TableExtDesc tableExtDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 7089eba..9837ff4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -18,13 +18,14 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import com.google.common.collect.Maps;
-public class TableRef {
+public class TableRef implements Serializable{
final private DataModelDesc model;
final private String alias;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
index 84e5fd5..421f802 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java
@@ -23,10 +23,12 @@ import org.apache.kylin.metadata.realization.RealizationType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
/**
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class RealizationEntry {
+public class RealizationEntry implements Serializable{
@JsonProperty("type")
private RealizationType type;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index d51ae9e..b50a7e4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -213,7 +213,7 @@ public class Tuple implements ITuple {
case "float":
return Float.valueOf(strValue);
case "boolean":
- return Boolean.valueOf(strValue) | "1".equals(strValue); // in some extended encodings boolean might be encoded as a number
+ return Boolean.valueOf(strValue) || "1".equals(strValue); // in some extended encodings boolean might be encoded as a number
default:
return strValue;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
index 8970124..c3e88b5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
@@ -93,10 +93,12 @@ public class TupleInfo {
else
columns.add(index, col);
- if (dataTypeNames.size() > index)
- dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName());
- else
- dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName());
+ if (col != null) {
+ if (dataTypeNames.size() > index)
+ dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName());
+ else
+ dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName());
+ }
}
public List<String> getAllFields() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
index 8ea96de..c62f1fe 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java
@@ -20,6 +20,7 @@ package org.apache.kylin.source;
import java.io.Closeable;
import java.io.IOException;
+import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -59,7 +60,7 @@ public interface ReadableTable {
// ============================================================================
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
- public class TableSignature {
+ public class TableSignature implements Serializable{
@JsonProperty("path")
private String path;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
index 7c54714..f086ae9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -195,7 +195,7 @@ public class ColumnGenerator {
private long randomMillis() {
int secondsInYear = 3600 * 24 * 365;
- long year = randStart + rand.nextInt(randEnd - randStart) - 1970;
+ long year = randStart + rand.nextInt(randEnd - randStart) - 1970L;
long second = year * secondsInYear + rand.nextInt(secondsInYear);
return second * 1000;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
index 67fedbd..847f4bf 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java
@@ -88,7 +88,7 @@ public class DFSFileTableReader implements TableReader {
try {
for (FileStatus f : allFiles) {
- RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString());
+ RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), f.getPath().toString());
this.readerList.add(rowReader);
}
} catch (IOException e) {
@@ -208,7 +208,7 @@ public class DFSFileTableReader implements TableReader {
Writable key;
Text value;
- SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException {
+ SeqRowReader(Configuration hconf, String path) throws IOException {
reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path)));
key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
value = new Text();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 7dcb73e..1107aa5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -150,6 +150,7 @@ public class HadoopStatusGetter {
logger.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
logger.error(e.getMessage());
} finally {
httpget.releaseConnection();
@@ -212,6 +213,7 @@ public class HadoopStatusGetter {
logger.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
logger.error(e.getMessage());
} finally {
get.releaseConnection();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index a26d4ff..be11e39 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -84,6 +84,7 @@ public class MapReduceExecutable extends AbstractExecutable {
logger.warn("error get hadoop status");
super.onExecuteStart(executableContext);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
logger.warn("error get hadoop status");
super.onExecuteStart(executableContext);
}
@@ -171,7 +172,7 @@ public class MapReduceExecutable extends AbstractExecutable {
return new ExecuteResult(ExecuteResult.State.FAILED, output.toString());
}
}
- Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000);
+ Thread.sleep(context.getConfig().getYarnStatusCheckIntervalSeconds() * 1000L);
}
// try to kill running map-reduce job to release resources.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
index aedf541..d183f90 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MapContextGTRecordWriter.java
@@ -41,6 +41,7 @@ public class MapContextGTRecordWriter extends KVGTRecordWriter {
try {
mapContext.write(key, value);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index b804eef..74525d3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -60,7 +60,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
byte[] data2 = o.text.getBytes();
String str1 = new String(data1, 1, data1.length - 1);
String str2 = new String(data2, 1, data2.length - 1);
- if (str1 == null || str1.equals("") || str2 == null || str2.equals("")) {
+ if (str1.equals("") || str2.equals("")) {
//should not achieve here
logger.error("none numeric value!");
return 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 6e894dd..4ec5048 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -167,6 +167,9 @@ public class SparkCubing extends AbstractApplication {
return false;
}
});
+ if (files == null) {
+ return;
+ }
for (File file : files) {
sc.addFile(file.getAbsolutePath());
}
@@ -341,6 +344,7 @@ public class SparkCubing extends AbstractApplication {
Dictionary<String> dict = cubeSegment.getDictionary(col);
if (dict == null) {
System.err.println("Dictionary for " + col + " was not found.");
+ continue;
}
dictionaryMap.put(col, dict);
System.out.println("col:" + col + " dictionary size:" + dict.getSize());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/BufferedCuboidWriter.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/BufferedCuboidWriter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/BufferedCuboidWriter.java
index 8cc0792..b3334b7 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/BufferedCuboidWriter.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/BufferedCuboidWriter.java
@@ -44,6 +44,7 @@ public class BufferedCuboidWriter implements SparkCuboidWriter {
try {
blockingQueue.put(tupleConverter.convert(cuboidId, record));
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
@@ -57,6 +58,7 @@ public class BufferedCuboidWriter implements SparkCuboidWriter {
try {
blockingQueue.put(new Tuple2(new byte[0], new byte[0]));
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
@@ -75,6 +77,7 @@ public class BufferedCuboidWriter implements SparkCuboidWriter {
try {
current = blockingQueue.take();
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index e75e683..1147008 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -126,13 +126,6 @@
<artifactId>aspectjweaver</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${kafka.version}</version>
- <scope>provided</scope>
- </dependency>
-
<!-- Test & Env -->
<dependency>
<groupId>org.apache.kylin</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
index 95bc2f5..e296248 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.request;
+import java.io.Serializable;
import java.util.Arrays;
/**
@@ -40,7 +41,7 @@ public class PrepareSqlRequest extends SQLRequest {
this.params = params;
}
- public static class StateParam {
+ public static class StateParam implements Serializable{
private String className;
private String value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 2b85e69..64f91b1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -52,7 +52,7 @@ public class BadQueryDetector extends Thread {
super("BadQueryDetector");
this.setDaemon(true);
this.kylinConfig = KylinConfig.getInstanceFromEnv();
- this.detectionInterval = kylinConfig.getBadQueryDefaultDetectIntervalSeconds() * 1000;
+ this.detectionInterval = kylinConfig.getBadQueryDefaultDetectIntervalSeconds() * 1000L;
this.alertMB = 100;
this.alertRunningSec = kylinConfig.getBadQueryDefaultAlertingSeconds();
@@ -118,6 +118,7 @@ public class BadQueryDetector extends Thread {
try {
Thread.sleep(detectionInterval);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
// stop detection and exit
return;
}
@@ -208,7 +209,7 @@ public class BadQueryDetector extends Thread {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user, Thread t) {
try {
- long cachingSeconds = (kylinConfig.getBadQueryDefaultAlertingSeconds() + 1) * 30;
+ long cachingSeconds = (kylinConfig.getBadQueryDefaultAlertingSeconds() + 1) * 30L;
Pair<Long, String> sqlPair = new Pair<>(startTime, sql);
if (!cacheQueue.contains(sqlPair)) {
badQueryManager.addEntryToProject(sql, startTime, adj, runningSec, serverHostname, t.getName(), user, project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 23aa5a4..d28c87c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -455,6 +455,10 @@ public class CubeService extends BasicService {
}
}
+ if(toDelete == null){
+ throw new IllegalArgumentException("Cannot find segment '" + segmentName +"'");
+ }
+
if (toDelete.getStatus() != SegmentStatusEnum.READY) {
throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as its status is not READY. Discard the on-going job for it.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
index dee5531..8208596 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
@@ -45,7 +45,11 @@ public class DiagnosisService extends BasicService {
}
private String getDiagnosisPackageName(File destDir) {
- for (File subDir : destDir.listFiles()) {
+ File[] files = destDir.listFiles();
+ if (files == null) {
+ throw new RuntimeException("Diagnosis package is not available in directory: " + destDir.getAbsolutePath());
+ }
+ for (File subDir : files) {
if (subDir.isDirectory()) {
for (File file : subDir.listFiles()) {
if (file.getName().endsWith(".zip")) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index f046f78..145ed31 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -80,8 +80,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
System.out.println("Get row " + counter + " column '" + field + "' value: " + fieldValue);
}
- if (fieldValue != null)
- getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
+ getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
}
counter++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index d59c257..e43894d 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -51,7 +51,6 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
</dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java
index add825b..fc3bba0 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java
@@ -21,10 +21,12 @@ package org.apache.kylin.source.kafka.config;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.Serializable;
+
/**
*/
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class BrokerConfig {
+public class BrokerConfig implements Serializable{
@JsonProperty("id")
private int id;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
index 06cdbba..2a85894 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java
@@ -29,7 +29,7 @@ import com.google.common.collect.Maps;
*/
@SuppressWarnings("rawtypes")
public abstract class AggregationCache {
- static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB
+ static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024L; // 500 MB
static final long MEMOERY_MAX_BYTES = Runtime.getRuntime().maxMemory();
protected final Map<AggrKey, MeasureAggregator[]> aggBufMap;
transient int rowMemBytes;
@@ -75,7 +75,7 @@ public abstract class AggregationCache {
}
}
int size = aggBufMap.size();
- long memUsage = (40 + rowMemBytes) * size;
+ long memUsage = (40L + rowMemBytes) * size;
if (memUsage > MEMORY_USAGE_CAP) {
throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 2d574bd..59fe9e0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -74,6 +74,7 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
return ret;
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException("Error when waiting queue", e);
}
}
@@ -87,6 +88,7 @@ class ExpectedSizeIterator implements Iterator<byte[]> {
try {
queue.put(data);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException("error when waiting queue", e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index e26c8e8..1db60fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -116,6 +116,7 @@ public class HbaseStreamingInput {
logger.warn("There are another " + waiting + " batches waiting to be added");
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
e.printStackTrace();
}
@@ -166,6 +167,7 @@ public class HbaseStreamingInput {
logger.warn("Too many queries to handle! Blocking " + waiting + " sets of scan requests");
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
e.printStackTrace();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index e219c5a..1ea8e8d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -51,6 +51,7 @@ public class PingHBaseCLI {
System.out.println("--------------Getting kerberos credential for user " + UserGroupInformation.getCurrentUser().getUserName());
TokenUtil.obtainAndCacheToken(hconf, UserGroupInformation.getCurrentUser());
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
System.out.println("--------------Error while getting kerberos credential for user " + UserGroupInformation.getCurrentUser().getUserName());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
index 613d783..5a44cc1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
@@ -141,6 +141,8 @@ public class ZookeeperDistributedJobLock implements DistributedJobLock {
} catch (Exception e) {
logger.error("fail to get the serverName for the path: " + lockPath, e);
}
+ if(lockServerName == null)
+ return false;
return lockServerName.equalsIgnoreCase(serverName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
index 40cbf32..b4c44c3 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
@@ -20,6 +20,7 @@ package org.apache.kylin.tool;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Set;
import org.apache.commons.cli.Option;
@@ -126,7 +127,7 @@ public class CubeMetaIngester extends AbstractApplication {
tempFolder.mkdir();
ZipFileUtils.decompressZipfileToDirectory(srcPath, tempFolder);
if (tempFolder.list().length != 1) {
- throw new IllegalStateException(tempFolder.list().toString());
+ throw new IllegalStateException(Arrays.toString(tempFolder.list()));
}
injest(tempFolder.listFiles()[0].getAbsoluteFile());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 94fccf8..e1f994f 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -231,6 +231,7 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new RuntimeException("Diagnosis info dump interrupted.", e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c64f55b2/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
index f993155..0d8c08f 100644
--- a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
@@ -51,7 +51,7 @@ import com.google.common.collect.Lists;
public class HBaseUsageExtractor extends AbstractInfoExtractor {
- private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class);
+ private static final Logger logger = LoggerFactory.getLogger(HBaseUsageExtractor.class);
@SuppressWarnings("static-access")
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
@SuppressWarnings("static-access")
[40/50] [abbrv] kylin git commit: KYLIN-2367 ammend test case
Posted by li...@apache.org.
KYLIN-2367 ammend test case
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07e5e5a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07e5e5a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07e5e5a7
Branch: refs/heads/master-hbase1.x
Commit: 07e5e5a75f4f692a53ac48c64c3cb44b84022c88
Parents: b0aadb1
Author: Li Yang <li...@apache.org>
Authored: Tue Jan 10 15:45:05 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 10 15:45:05 2017 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/07e5e5a7/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 2b33e66..df53de9 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -48,7 +48,7 @@
}, {
"id" : "10",
"name" : "ITEM_COUNT",
- "datatype" : "int",
+ "datatype" : "bigint",
"data_gen" : "RAND"
}, {
"id" : "11",
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
- private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -386,7 +387,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 61c73d5..1cdb2f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -235,9 +236,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -257,7 +258,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index 1db60fb..8dd2164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -120,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -136,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -172,8 +172,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -208,7 +208,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -226,8 +226,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 1ea8e8d..bba6745 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -59,12 +60,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index d1a74ad..f3d7649 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -57,6 +59,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,7 +80,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -153,22 +157,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 5269195..48d3be8 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -231,6 +231,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -447,11 +448,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -471,7 +472,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -537,13 +537,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -560,7 +559,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c47f4ad5/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 19e5db0..f52fc3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[42/50] [abbrv] kylin git commit: KYLIN-2376 upgrade checkstyle plugin
Posted by li...@apache.org.
KYLIN-2376 upgrade checkstyle plugin
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/521dfffa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/521dfffa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/521dfffa
Branch: refs/heads/master-hbase1.x
Commit: 521dfffae15d0c85885ec816a4fc40907bfb7645
Parents: c84b46b
Author: etherge <et...@163.com>
Authored: Tue Jan 10 19:09:58 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Jan 10 20:53:57 2017 +0800
----------------------------------------------------------------------
pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/521dfffa/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fcb5ce3..e391103 100644
--- a/pom.xml
+++ b/pom.xml
@@ -907,6 +907,13 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>2.17</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>6.19</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>check-style</id>
[23/50] [abbrv] kylin git commit: KYLIN-2353 Serialize BitmapCounter
with distinct count
Posted by li...@apache.org.
KYLIN-2353 Serialize BitmapCounter with distinct count
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/731a53a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/731a53a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/731a53a5
Branch: refs/heads/master-hbase1.x
Commit: 731a53a51a32fb1314b4e1282f158e3b7ed819a5
Parents: aa57446
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 31 20:28:30 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:08:37 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 105 ++++++++++++-------
.../bitmap/BitmapDistinctCountAggFunc.java | 2 +-
.../kylin/measure/bitmap/BitmapSerializer.java | 15 +--
3 files changed, 70 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index aeb14ba..a18ac4a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -35,6 +35,8 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
private final int VERSION = 2;
+ private Integer count;
+ private ByteBuffer buffer;
public BitmapCounter() {
}
@@ -43,18 +45,41 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
merge(another);
}
+ private MutableRoaringBitmap getBitmap() {
+ if (!bitmap.isEmpty()) {
+ return bitmap;
+ }
+
+ if (buffer != null) {
+ int version = buffer.getInt();
+ int size = buffer.getInt();
+ count = buffer.getInt();
+
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(buffer))) {
+ bitmap.deserialize(is);
+ } catch (IOException e) {
+ throw new RuntimeException("deserialize bitmap failed!");
+ }
+
+ buffer = null;
+ }
+
+ return bitmap;
+ }
+
public void clear() {
- bitmap.clear();
+ getBitmap().clear();
}
public BitmapCounter clone() {
BitmapCounter newCounter = new BitmapCounter();
- newCounter.bitmap = bitmap.clone();
+ newCounter.bitmap = getBitmap().clone();
return newCounter;
}
public void add(int value) {
- bitmap.add(value);
+ getBitmap().add(value);
+ count = getBitmap().getCardinality();
}
public void add(byte[] value) {
@@ -77,34 +102,43 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
public void merge(BitmapCounter another) {
- this.bitmap.or(another.bitmap);
+ getBitmap().or(another.getBitmap());
+ count = getBitmap().getCardinality();
}
public void intersect(BitmapCounter another) {
- this.bitmap.and(another.bitmap);
+ getBitmap().and(another.getBitmap());
+ count = getBitmap().getCardinality();
}
- public long getCount() {
- return this.bitmap.getCardinality();
+ public int getCount() {
+ if (count != null) {
+ return count;
+ }
+
+ return getBitmap().getCardinality();
}
public int getMemBytes() {
- return this.bitmap.getSizeInBytes();
+ return getBitmap().getSizeInBytes();
}
public Iterator<Integer> iterator() {
- return bitmap.iterator();
+ return getBitmap().iterator();
}
public void writeRegisters(ByteBuffer out) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
+ MutableRoaringBitmap bitmap = getBitmap();
bitmap.runOptimize();
bitmap.serialize(dos);
dos.close();
ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+
out.putInt(VERSION);
- out.putInt(bos.size() + 4 + 4);
+ out.putInt(bos.size() + 4 + 4 + 4);
+ out.putInt(getCount());
out.put(bb);
}
@@ -116,13 +150,30 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
if (version == VERSION) {
@SuppressWarnings("unused")
int size = in.getInt();
+ count = in.getInt();
+ in.position(mark);
+ buffer = cloneBuffer(in, size);
} else {
in.position(mark);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ getBitmap().deserialize(is);
+ }
}
+ }
- try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
- bitmap.deserialize(is);
- }
+ private ByteBuffer cloneBuffer(ByteBuffer src, int size) throws IOException {
+ int mark = src.position();
+ int limit = src.limit();
+
+ src.limit(mark + size);
+ ByteBuffer clone = ByteBuffer.allocate(size);
+ clone.put(src.slice());
+ clone.flip();
+
+ src.position(mark + size);
+ src.limit(limit);
+
+ return clone;
}
public int peekLength(ByteBuffer in) {
@@ -132,7 +183,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
// keep forward compatibility
if (version == VERSION) {
- len = in.getInt() ;
+ len = in.getInt();
} else {
in.position(mark);
try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
@@ -149,32 +200,10 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
@Override
- public String toString() {
- long count = getCount();
- if (count <= 10) {
- return "(" + count + ")" + bitmap.toString();
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append("(").append(count).append("){");
- int values = 0;
- for (Integer v : bitmap) {
- if (values++ < 10) {
- sb.append(v).append(",");
- } else {
- sb.append("...");
- break;
- }
- }
- sb.append("}");
- return sb.toString();
- }
- }
-
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + bitmap.hashCode();
+ result = prime * result + getBitmap().hashCode();
return result;
}
@@ -187,7 +216,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
if (getClass() != obj.getClass())
return false;
BitmapCounter other = (BitmapCounter) obj;
- return bitmap.equals(other.bitmap);
+ return getBitmap().equals(other.getBitmap());
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
index d039b6d..3a1a800 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
@@ -35,7 +35,7 @@ public class BitmapDistinctCountAggFunc {
public static BitmapCounter add(BitmapCounter counter, Object v) {
BitmapCounter c = (BitmapCounter) v;
if (counter == null) {
- return new BitmapCounter(c);
+ return c;
} else {
counter.merge(c);
return counter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
index 089d18c..4890295 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
@@ -29,8 +29,6 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
*/
public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
- private ThreadLocal<BitmapCounter> current = new ThreadLocal<>();
-
public BitmapSerializer(DataType type) {
}
@@ -43,18 +41,9 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
}
}
- private BitmapCounter current() {
- BitmapCounter counter = current.get();
- if (counter == null) {
- counter = new BitmapCounter();
- current.set(counter);
- }
- return counter;
- }
-
@Override
public BitmapCounter deserialize(ByteBuffer in) {
- BitmapCounter counter = current();
+ BitmapCounter counter = new BitmapCounter();
try {
counter.readRegisters(in);
} catch (IOException e) {
@@ -65,7 +54,7 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public int peekLength(ByteBuffer in) {
- return current().peekLength(in);
+ return new BitmapCounter().peekLength(in);
}
@Override
[44/50] [abbrv] kylin git commit: KYLIN-2380 refactor DbUnit assertion
Posted by li...@apache.org.
KYLIN-2380 refactor DbUnit assertion
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5f6450d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5f6450d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5f6450d6
Branch: refs/heads/master-hbase1.x
Commit: 5f6450d6e77f9f62c3670c20ec4d8aef18bfd2fb
Parents: 78d7086
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 11 12:10:36 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Jan 11 12:39:24 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/query/HackedDbUnitAssert.java | 134 ++++++++++++++++---
.../apache/kylin/query/ITMassInQueryTest.java | 9 +-
.../org/apache/kylin/query/KylinTestBase.java | 65 +++++----
3 files changed, 152 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
index c295430..3a21570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java
@@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns;
import org.dbunit.dataset.DataSetException;
import org.dbunit.dataset.ITable;
import org.dbunit.dataset.ITableMetaData;
+import org.dbunit.dataset.datatype.BigIntegerDataType;
import org.dbunit.dataset.datatype.DataType;
+import org.dbunit.dataset.datatype.IntegerDataType;
+import org.dbunit.dataset.datatype.UnknownDataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * dirty hack to support checking result of SQL with limit
+ * A copy of DbUnitAssert, in which a few hacks applied
+ *
+ * - tolerate some column type difference, like INT vs. BIGINT
+ * - check expected table contains actual table (instead of equals), for sql with limit
*/
public class HackedDbUnitAssert extends DbUnitAssert {
private static final Logger logger = LoggerFactory.getLogger(HackedDbUnitAssert.class);
+ private boolean hackCheckContains;
+ private boolean hackIgnoreIntBigIntMismatch;
+
+ public void hackCheckContains() {
+ hackCheckContains = true;
+ }
+
+ public void hackIgnoreIntBigIntMismatch() {
+ hackIgnoreIntBigIntMismatch = true;
+ }
+
+ // THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY hackXXX CONDITION CHECKS.
public void assertEquals(ITable expectedTable, ITable actualTable, FailureHandler failureHandler) throws DatabaseUnitException {
logger.trace("assertEquals(expectedTable, actualTable, failureHandler) - start");
logger.debug("assertEquals: expectedTable={}", expectedTable);
@@ -57,25 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert {
ITableMetaData actualMetaData = actualTable.getTableMetaData();
String expectedTableName = expectedMetaData.getTableName();
- // // Verify row count
- // int expectedRowsCount = expectedTable.getRowCount();
- // int actualRowsCount = actualTable.getRowCount();
- // if (expectedRowsCount != actualRowsCount) {
- // String msg = "row count (table=" + expectedTableName + ")";
- // Error error =
- // failureHandler.createFailure(msg, String
- // .valueOf(expectedRowsCount), String
- // .valueOf(actualRowsCount));
- // logger.error(error.toString());
- // throw error;
- // }
-
- // if both tables are empty, it is not necessary to compare columns, as
- // such
- // comparison
- // can fail if column metadata is different (which could occurs when
- // comparing empty tables)
- if (expectedTable.getRowCount() == 0 && actualTable.getRowCount() == 0) {
+ // Verify row count
+ int expectedRowsCount = expectedTable.getRowCount();
+ int actualRowsCount = actualTable.getRowCount();
+ if (!hackCheckContains) {
+ if (expectedRowsCount != actualRowsCount) {
+ String msg = "row count (table=" + expectedTableName + ")";
+ Error error = failureHandler.createFailure(msg, String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount));
+ logger.error(error.toString());
+ throw error;
+ }
+ }
+
+ // if both tables are empty, it is not necessary to compare columns, as such comparison
+ // can fail if column metadata is different (which could occurs when comparing empty tables)
+ if (expectedRowsCount == 0 && actualRowsCount == 0) {
logger.debug("Tables are empty, hence equals.");
return;
}
@@ -97,10 +111,86 @@ public class HackedDbUnitAssert extends DbUnitAssert {
ComparisonColumn[] comparisonCols = getComparisonColumns(expectedTableName, expectedColumns, actualColumns, failureHandler);
// Finally compare the data
- compareData(expectedTable, actualTable, comparisonCols, failureHandler);
+ if (hackCheckContains)
+ compareDataContains(expectedTable, actualTable, comparisonCols, failureHandler);
+ else
+ compareData(expectedTable, actualTable, comparisonCols, failureHandler);
+ }
+
+ // THIS METHOD IS COPIED FROM SUPER CLASS TO CHANGE ComparisonColumn TO OUR OWN.
+ @Override
+ protected ComparisonColumn[] getComparisonColumns(String expectedTableName, Column[] expectedColumns, Column[] actualColumns, FailureHandler failureHandler) {
+ ComparisonColumn[] result = new ComparisonColumn[expectedColumns.length];
+
+ for (int j = 0; j < expectedColumns.length; j++) {
+ Column expectedColumn = expectedColumns[j];
+ Column actualColumn = actualColumns[j];
+ result[j] = new HackedComparisonColumn(expectedTableName, expectedColumn, actualColumn, failureHandler);
+ }
+ return result;
+ }
+
+ // MOSTLY COPIED FROM SUPER CLASS
+ private class HackedComparisonColumn extends ComparisonColumn {
+ private String columnName;
+ private DataType dataType;
+
+ public HackedComparisonColumn(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) {
+
+ // super class is actually useless, all public methods are overridden below
+ super(tableName, expectedColumn, expectedColumn, failureHandler);
+
+ this.columnName = expectedColumn.getColumnName();
+ this.dataType = getComparisonDataType(tableName, expectedColumn, actualColumn, failureHandler);
+ }
+
+ @Override
+ public String getColumnName() {
+ return this.columnName;
+ }
+
+ @Override
+ public DataType getDataType() {
+ return this.dataType;
+ }
+
+ // COPIED FROM SUPER CLASS, CHANGES ARE LEAD BY hackXXX CONDITION CHECKS.
+ private DataType getComparisonDataType(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) {
+ if (logger.isDebugEnabled())
+ logger.debug("getComparisonDataType(tableName={}, expectedColumn={}, actualColumn={}, failureHandler={}) - start", new Object[] { tableName, expectedColumn, actualColumn, failureHandler });
+
+ DataType expectedDataType = expectedColumn.getDataType();
+ DataType actualDataType = actualColumn.getDataType();
+
+ // The two columns have different data type
+ if (!expectedDataType.getClass().isInstance(actualDataType)) {
+ // Expected column data type is unknown, use actual column data type
+ if (expectedDataType instanceof UnknownDataType) {
+ return actualDataType;
+ }
+
+ // Actual column data type is unknown, use expected column data type
+ if (actualDataType instanceof UnknownDataType) {
+ return expectedDataType;
+ }
+
+ if (hackIgnoreIntBigIntMismatch) {
+ if (expectedDataType instanceof IntegerDataType && actualDataType instanceof BigIntegerDataType)
+ return actualDataType;
+ }
+
+ // Impossible to determine which data type to use
+ String msg = "Incompatible data types: (table=" + tableName + ", col=" + expectedColumn.getColumnName() + ")";
+ throw failureHandler.createFailure(msg, String.valueOf(expectedDataType), String.valueOf(actualDataType));
+ }
+
+ // Both columns have same data type, return any one of them
+ return expectedDataType;
+ }
+
}
- protected void compareData(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException {
+ private void compareDataContains(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException {
logger.debug("compareData(expectedTable={}, actualTable={}, " + "comparisonCols={}, failureHandler={}) - start", new Object[] { expectedTable, actualTable, comparisonCols, failureHandler });
if (expectedTable == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index fd3abfb..28cdb67 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -30,12 +30,9 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.dbunit.Assertion;
-import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.dbunit.dataset.ITable;
-import org.dbunit.ext.h2.H2Connection;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -144,13 +141,11 @@ public class ITMassInQueryTest extends KylinTestBase {
sql = sql.replace("massin(test_kylin_fact.SELLER_ID,'vip_customers')", "test_kylin_fact.SELLER_ID in ( " + org.apache.commons.lang.StringUtils.join(vipSellers, ",") + ")");
printInfo("Query Result from H2 - " + queryName);
printInfo("Query for H2 - " + sql);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5f6450d6/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index ae69f09..a8eb505 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -51,7 +51,7 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.dbunit.Assertion;
+import org.dbunit.DatabaseUnitException;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
@@ -229,11 +229,13 @@ public class KylinTestBase {
sql = changeJoinType(sql, joinType);
ITable queryTable = dbConn.createQueryTable(resultTableName + queryName, sql);
- String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length];
- for (int i = 0; i < columnNames.length; i++) {
- columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName();
- }
if (needSort) {
+ String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length];
+ for (int i = 0; i < columnNames.length; i++) {
+ columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName();
+ }
+ Arrays.sort(columnNames);
+
queryTable = new SortedTable(queryTable, columnNames);
}
if (PRINT_RESULT)
@@ -333,10 +335,7 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
- executeQuery(h2Conn, queryName, sql, needSort);
+ executeQuery(newH2Connection(), queryName, sql, needSort);
}
}
@@ -358,7 +357,7 @@ public class KylinTestBase {
// compare the result
Assert.assertEquals(queryName, expectRowCount, kylinTable.getRowCount());
- // Assertion.assertEquals(expectRowCount, kylinTable.getRowCount());
+ // assertTableEquals(expectRowCount, kylinTable.getRowCount());
}
}
@@ -381,7 +380,7 @@ public class KylinTestBase {
ITable kylinTable = executeQuery(kylinConn, queryName, sql, false);
// compare the result
- Assertion.assertEquals(expectTable, kylinTable);
+ assertTableEquals(expectTable, kylinTable);
}
}
@@ -404,9 +403,7 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
@@ -467,13 +464,10 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, false);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, false);
try {
- HackedDbUnitAssert hackedDbUnitAssert = new HackedDbUnitAssert();
- hackedDbUnitAssert.assertEquals(h2Table, kylinTable);
+ assertTableContains(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
@@ -506,13 +500,11 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- H2Connection h2Conn = new H2Connection(h2Connection, null);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort);
+ ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort);
try {
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
} catch (Throwable t) {
printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath());
throw t;
@@ -545,14 +537,33 @@ public class KylinTestBase {
// execute H2
printInfo("Query Result from H2 - " + queryName);
- IDatabaseConnection h2Conn = new DatabaseConnection(h2Connection);
- h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
- ITable h2Table = executeDynamicQuery(h2Conn, queryName, sql, parameters, needSort);
+ ITable h2Table = executeDynamicQuery(newH2Connection(), queryName, sql, parameters, needSort);
// compare the result
- Assertion.assertEquals(h2Table, kylinTable);
+ assertTableEquals(h2Table, kylinTable);
}
}
+
+ protected void assertTableEquals(ITable h2Table, ITable kylinTable) throws DatabaseUnitException {
+ HackedDbUnitAssert dbUnit = new HackedDbUnitAssert();
+ dbUnit.hackIgnoreIntBigIntMismatch();
+ dbUnit.assertEquals(h2Table, kylinTable);
+ }
+
+ protected void assertTableContains(ITable h2Table, ITable kylinTable) throws DatabaseUnitException {
+ HackedDbUnitAssert dbUnit = new HackedDbUnitAssert();
+ dbUnit.hackIgnoreIntBigIntMismatch();
+ dbUnit.hackCheckContains();
+ dbUnit.assertEquals(h2Table, kylinTable);
+ }
+
+ @SuppressWarnings("deprecation")
+ protected static H2Connection newH2Connection() throws DatabaseUnitException {
+ H2Connection h2Conn = new H2Connection(h2Connection, null);
+ h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory());
+ h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false);
+ return h2Conn;
+ }
protected int runSqlFile(String file) throws Exception {
return runSQL(new File(file), true, false);
[41/50] [abbrv] kylin git commit: KYLIN-2379 Add
UseCMSInitiatingOccupancyOnly to KYLIN_JVM_SETTINGS
Posted by li...@apache.org.
KYLIN-2379 Add UseCMSInitiatingOccupancyOnly to KYLIN_JVM_SETTINGS
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c84b46bd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c84b46bd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c84b46bd
Branch: refs/heads/master-hbase1.x
Commit: c84b46bd5b72006f140f7aad147ba0185c7eecbb
Parents: 07e5e5a
Author: kangkaisen <ka...@live.com>
Authored: Tue Jan 10 20:32:39 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Jan 10 20:48:43 2017 +0800
----------------------------------------------------------------------
build/bin/setenv.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c84b46bd/build/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
index f6cf210..d838362 100755
--- a/build/bin/setenv.sh
+++ b/build/bin/setenv.sh
@@ -22,7 +22,7 @@
# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
# uncomment following to for it to take effect
export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
# uncomment following to for it to take effect(the values need adjusting to fit your env)
# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
[11/50] [abbrv] kylin git commit: KYLIN-2351 fix checkstyle
Posted by li...@apache.org.
KYLIN-2351 fix checkstyle
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f5734861
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f5734861
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f5734861
Branch: refs/heads/master-hbase1.x
Commit: f5734861863c506510ed23bc1b3891d566b7219d
Parents: 399f0c4
Author: Billy Liu <bi...@apache.org>
Authored: Fri Jan 6 09:15:38 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Jan 6 09:15:38 2017 +0800
----------------------------------------------------------------------
.../src/test/java/org/apache/kylin/query/ITMassInQueryTest.java | 1 -
.../src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java | 1 -
2 files changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f5734861/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
index 18d79ae..fd3abfb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java
@@ -26,7 +26,6 @@ import java.util.Random;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f5734861/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
index 786d7d1..605b79e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHdfsOpsTest.java
@@ -20,7 +20,6 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
[20/50] [abbrv] kylin git commit: fix NPE bug
Posted by li...@apache.org.
fix NPE bug
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a18af04
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a18af04
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a18af04
Branch: refs/heads/master-hbase1.x
Commit: 5a18af04ca40aaa6509cdf7c39b71b17dc5eb4a3
Parents: 122b15a
Author: xiefan46 <95...@qq.com>
Authored: Thu Jan 5 23:29:43 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Jan 7 10:18:20 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/CacheDictionary.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a18af04/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index 1e260b2..d7ed6bd 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHashMap;
public abstract class CacheDictionary<T> extends Dictionary<T> {
private static final long serialVersionUID = 1L;
- protected transient boolean enableValueCache = false;
-
private transient SoftReference<ConcurrentHashMap> valueToIdCache;
private transient SoftReference<Object[]> idToValueCache;
@@ -46,7 +44,7 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
//value --> id
@Override
protected final int getIdFromValueImpl(T value, int roundingFlag) {
- if (enableValueCache && roundingFlag == 0) {
+ if (this.valueToIdCache != null && roundingFlag == 0) {
Map cache = valueToIdCache.get(); // SoftReference to skip cache gracefully when short of memory
if (cache != null) {
Integer id;
@@ -66,7 +64,7 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
//id --> value
@Override
protected final T getValueFromIdImpl(int id) {
- if (enableValueCache) {
+ if (this.idToValueCache != null) {
Object[] cache = idToValueCache.get();
if (cache != null) {
int seq = calcSeqNoFromId(id);
@@ -91,7 +89,6 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
}
public final void enableCache() {
- this.enableValueCache = true;
if (this.valueToIdCache == null)
this.valueToIdCache = new SoftReference<>(new ConcurrentHashMap());
if (this.idToValueCache == null)
@@ -99,7 +96,6 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
}
public final void disableCache() {
- this.enableValueCache = false;
this.valueToIdCache = null;
this.idToValueCache = null;
}
[25/50] [abbrv] kylin git commit: KYLIN-2364 Output table name to
error info in LookupTable
Posted by li...@apache.org.
KYLIN-2364 Output table name to error info in LookupTable
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e46d699e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e46d699e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e46d699e
Branch: refs/heads/master-hbase1.x
Commit: e46d699e05100db084db354d7efb3786575d5c54
Parents: 56daf6d
Author: kangkaisen <ka...@live.com>
Authored: Sat Jan 7 19:43:43 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Sun Jan 8 12:14:05 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/lookup/LookupTable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e46d699e/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 21221f4..90ca500 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -83,7 +83,7 @@ abstract public class LookupTable<T> {
Array<T> key = new Array<T>(keyCols);
if (data.containsKey(key))
- throw new IllegalStateException("Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));
+ throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value));
data.put(key, value);
}
[08/50] [abbrv] kylin git commit: KYLIN-2356 Incorrect result when
filter on numeric columns
Posted by li...@apache.org.
KYLIN-2356 Incorrect result when filter on numeric columns
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/980b03b1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/980b03b1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/980b03b1
Branch: refs/heads/master-hbase1.x
Commit: 980b03b1abb8af82a9bceeb497847fe17c19c3e3
Parents: c64f55b
Author: zhengdong <zh...@outlook.com>
Authored: Thu Jan 5 15:05:35 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 5 17:45:20 2017 +0800
----------------------------------------------------------------------
.../filter/FilterCodeSystemFactory.java | 98 ++++++++++++++++++++
.../translate/DerivedFilterTranslator.java | 4 +-
2 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
new file mode 100755
index 0000000..bae8cf9
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterCodeSystemFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.filter;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+
+/**
+ * Created by donald.zheng on 2016/12/19.
+ */
+public class FilterCodeSystemFactory {
+ private final static HashMap<String, IFilterCodeSystem> codeSystemMap = new HashMap<>();
+
+ static {
+ codeSystemMap.put("string", StringCodeSystem.INSTANCE);
+ codeSystemMap.put("integer", new IntegerCodeSystem());
+ codeSystemMap.put("decimal", new DecimalCodeSystem());
+ }
+
+ public static IFilterCodeSystem getFilterCodeSystem(DataType dataType) {
+ if (dataType.isIntegerFamily()) {
+ return codeSystemMap.get("integer");
+ } else if (dataType.isNumberFamily()) {
+ return codeSystemMap.get("decimal");
+ } else {
+ return codeSystemMap.get("string");
+ }
+ }
+
+ private static class IntegerCodeSystem implements IFilterCodeSystem {
+
+ @Override
+ public boolean isNull(Object code) {
+ return code == null;
+ }
+
+ @Override
+ public void serialize(Object code, ByteBuffer buf) {
+ BytesUtil.writeLong(Long.parseLong(code.toString()), buf);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buf) {
+ return BytesUtil.readLong(buf);
+ }
+
+ @Override
+ public int compare(Object o, Object t1) {
+ long l1 = Long.parseLong(o.toString());
+ long l2 = Long.parseLong(t1.toString());
+ return Long.compare(l1, l2);
+ }
+ }
+
+ private static class DecimalCodeSystem implements IFilterCodeSystem {
+ @Override
+ public boolean isNull(Object code) {
+ return code == null;
+ }
+
+ @Override
+ public void serialize(Object code, ByteBuffer buf) {
+ BytesUtil.writeUTFString(code.toString(), buf);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer buf) {
+ return Double.parseDouble(BytesUtil.readUTFString(buf));
+ }
+
+ @Override
+ public int compare(Object o, Object t1) {
+ double d1 = Double.parseDouble(o.toString());
+ double d2 = Double.parseDouble(t1.toString());
+ return Double.compare(d1, d2);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b03b1/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
old mode 100644
new mode 100755
index 13c655c..f4150fe
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -31,8 +31,8 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.FilterCodeSystemFactory;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -78,7 +78,7 @@ public class DerivedFilterTranslator {
SingleColumnTuple tuple = new SingleColumnTuple(derivedCol);
for (String[] row : lookup.getAllRows()) {
tuple.value = row[di];
- if (compf.evaluate(tuple, StringCodeSystem.INSTANCE)) {
+ if (compf.evaluate(tuple, FilterCodeSystemFactory.getFilterCodeSystem(derivedCol.getColumnDesc().getType()))) {
collect(row, pi, satisfyingHostRecords);
}
}