You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/27 07:28:34 UTC
[02/10] incubator-kylin git commit: Merge branch 'streaming' of
https://github.com/KylinOLAP/Kylin into streaming
Merge branch 'streaming' of https://github.com/KylinOLAP/Kylin into streaming
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/37d3fe3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/37d3fe3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/37d3fe3d
Branch: refs/heads/streaming
Commit: 37d3fe3d0fd5dc7f9a5d91f42f64ba85202c49d1
Parents: eea5b09
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Fri Feb 13 15:35:23 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Fri Feb 13 15:36:03 2015 +0800
----------------------------------------------------------------------
bin/check-env.sh | 20 ++--------
bin/find-hive-dependency.sh | 18 ++++++++-
bin/start-kylin.sh | 3 +-
bin/stop-kylin.sh | 3 ++
.../org/apache/kylin/common/KylinConfig.java | 13 ++++---
.../apache/kylin/common/util/HiveClient.java | 31 ++++++++++++++--
.../localmeta/hybrid/test_kylin_hybrid.json | 2 -
.../kylin/job/engine/JobEngineConfig.java | 24 +++++-------
.../kylin/job/hadoop/AbstractHadoopJob.java | 39 ++++++++++++++------
.../cardinality/HiveColumnCardinalityJob.java | 22 ++---------
.../kylin/job/hadoop/cube/CubeHFileJob.java | 7 +---
.../apache/kylin/job/hadoop/cube/CuboidJob.java | 7 +---
.../job/hadoop/cube/FactDistinctColumnsJob.java | 15 ++------
.../job/hadoop/cube/KeyDistributionJob.java | 2 +-
.../kylin/job/hadoop/cube/MergeCuboidJob.java | 8 +---
.../hadoop/cube/RangeKeyDistributionJob.java | 7 +---
.../cube/RowKeyDistributionCheckerJob.java | 2 +-
.../hadoop/invertedindex/IICreateHFileJob.java | 14 ++-----
.../invertedindex/IIDistinctColumnsJob.java | 18 +++------
.../hadoop/invertedindex/InvertedIndexJob.java | 16 +++-----
.../invertedindex/RandomKeyDistributionJob.java | 3 +-
script/compress.sh | 3 ++
script/package.sh | 8 ++--
.../kylin/storage/hybrid/HybridInstance.java | 13 ++++++-
.../kylin/storage/hybrid/HybridManager.java | 2 -
.../storage/hybrid/HybridStorageEngine.java | 31 +++++++++++++++-
.../kylin/storage/hybrid/HybridManagerTest.java | 2 +-
27 files changed, 172 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/bin/check-env.sh b/bin/check-env.sh
index 3bef6c4..14eea70 100644
--- a/bin/check-env.sh
+++ b/bin/check-env.sh
@@ -1,39 +1,27 @@
#!/bin/sh
+#by source
-echo "Checking KYLIN_HOME..."
if [ -z "$KYLIN_HOME" ]
then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
-else
- echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+ bin=$(dirname ${0})
+ export KYLIN_HOME=$(dirname $bin)
fi
+echo "KYLIN_HOME is set to ${KYLIN_HOME}"
-echo "Checking hbase..."
if [ -z "$(command -v hbase version)" ]
then
echo "Please make sure the user has the privilege to run hbase shell"
exit 1
-else
- echo "hbase check passed"
fi
-echo "Checking hive..."
if [ -z "$(command -v hive --version)" ]
then
echo "Please make sure the user has the privilege to run hive shell"
exit 1
-else
- echo "hive check passed"
fi
-echo "Checking hadoop..."
if [ -z "$(command -v hadoop version)" ]
then
echo "Please make sure the user has the privilege to run hadoop shell"
exit 1
-else
- echo "hadoop check passed"
fi
-
-exit 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hive-dependency.sh b/bin/find-hive-dependency.sh
index 7bd5f43..f15e87e 100644
--- a/bin/find-hive-dependency.sh
+++ b/bin/find-hive-dependency.sh
@@ -4,17 +4,31 @@ hive_env=`hive -e set | grep 'env:CLASSPATH'`
hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
arr=(`echo $hive_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hive_conf_path=
hive_exec_path=
for data in ${arr[@]}
do
- result=`echo $data | grep 'hive-exec.jar'`
+ result=`echo $data | grep -e 'hive-exec[0-9\.-]*jar'`
if [ $result ]
then
hive_exec_path=$data
fi
+ result=`echo $data | grep 'hive/conf'`
+ if [ $result ]
+ then
+ hive_conf_path=$data
+ fi
done
hdp_home=`echo $hive_exec_path | awk -F '/hive/lib/' '{print $1}'`
-hive_dependency=/usr/hdp/current/hive-client/conf/:${hdp_home}/hive/lib/*:${hdp_home}/hive-hcatalog/share/hcatalog/*
+hcatalog=`find $hdp_home -name "hive-hcatalog-core[0-9\.-]*jar" 2>&1 | grep -m 1 -v 'Permission denied'`
+
+if [ -z "$hcatalog" ]
+then
+ echo "hcatalog lib not found"
+ exit 1
+fi
+
+hive_dependency=${hive_conf_path}:${hdp_home}/hive/lib/*:${hcatalog}
echo "hive dependency: $hive_dependency"
export hive_dependency
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/bin/start-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/start-kylin.sh b/bin/start-kylin.sh
index b2544a5..70cbb62 100644
--- a/bin/start-kylin.sh
+++ b/bin/start-kylin.sh
@@ -1,12 +1,11 @@
#!/bin/sh
dir=$(dirname ${0})
+source ${dir}/check-env.sh
tomcat_root=${dir}/../tomcat
export tomcat_root
-sh ${dir}/check-env.sh || { exit 1; }
-
#if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
#then
# echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/bin/stop-kylin.sh
----------------------------------------------------------------------
diff --git a/bin/stop-kylin.sh b/bin/stop-kylin.sh
index c7a61b2..dad0a83 100644
--- a/bin/stop-kylin.sh
+++ b/bin/stop-kylin.sh
@@ -1,5 +1,8 @@
#!/bin/sh
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
if [ ! -f "${KYLIN_HOME}/pid" ]
then
echo "no pid found"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 5138c0b..1a94bff 100644
--- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -413,7 +413,7 @@ public class KylinConfig {
public String[] getRestServers() {
String nodes = getOptional(KYLIN_REST_SERVERS);
- if (nodes == null)
+ if (StringUtils.isBlank(nodes))
return null;
return nodes.split("\\s*,\\s*");
}
@@ -515,7 +515,7 @@ public class KylinConfig {
public static String getKylinHome() {
String kylinHome = System.getenv(KYLIN_HOME);
if (StringUtils.isEmpty(kylinHome)) {
- logger.warn("KYLIN_HOME has not been set");
+ logger.warn("KYLIN_HOME was not set");
return kylinHome;
}
return kylinHome;
@@ -531,15 +531,16 @@ public class KylinConfig {
private static File getKylinProperties() {
String kylinConfHome = System.getProperty(KYLIN_CONF_HOME);
- if (StringUtils.isEmpty(kylinConfHome)) {
- logger.warn("KYLIN_CONF_HOME has not been set");
- } else {
+ if (!StringUtils.isEmpty(kylinConfHome)) {
+ logger.info("Use KYLIN_CONF_HOME=" + kylinConfHome);
return getKylinPropertiesFile(kylinConfHome);
}
+ logger.warn("KYLIN_CONF_HOME property was not set, will seek KYLIN_HOME env variable");
+
String kylinHome = getKylinHome();
if (StringUtils.isEmpty(kylinHome))
- throw new RuntimeException("getKylinProperties needs KYLIN_HOME");
+ throw new RuntimeException("Didn't find KYLIN_CONF_HOME or KYLIN_HOME, please set one of them");
String path = kylinHome + File.separator + "conf";
return getKylinPropertiesFile(path);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
index 2588257..cfeea31 100644
--- a/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
+++ b/common/src/main/java/org/apache/kylin/common/util/HiveClient.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.stats.StatsUtils;
/**
* Hive meta API client for Kylin
@@ -94,7 +93,7 @@ public class HiveClient {
CommandProcessorResponse response = getDriver().run(hql);
int retCode = response.getResponseCode();
if (retCode != 0) {
- String err = response.getErrorMessage();
+ String err = response.getErrorMessage();
throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
}
}
@@ -125,11 +124,35 @@ public class HiveClient {
}
public long getFileSizeForTable(Table table) {
- return StatsUtils.getTotalSize(new org.apache.hadoop.hive.ql.metadata.Table(table));
+ return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE);
}
public long getFileNumberForTable(Table table) {
- return StatsUtils.getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
+ return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
+ }
+
+ /**
+ * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
+ *
+ * Get basic stats of table
+ * @param table
+ * - table
+ * @param statType
+ * - type of stats
+ * @return value of stats
+ */
+ public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
+ Map<String, String> params = table.getParameters();
+ long result = 0;
+
+ if (params != null) {
+ try {
+ result = Long.parseLong(params.get(statType));
+ } catch (NumberFormatException e) {
+ result = 0;
+ }
+ }
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid.json b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid.json
index a11b518..c5d4f91 100644
--- a/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid.json
+++ b/examples/test_case_data/localmeta/hybrid/test_kylin_hybrid.json
@@ -2,12 +2,10 @@
"uuid": "5ca78590-64b6-4367-8fb5-7500eb95fd9c",
"name": "test_kylin_hybrid",
"historyRealization": {
- "name": "test_kylin_cube_with_slr_left_join_empty",
"type": "CUBE",
"realization": "test_kylin_cube_with_slr_left_join_empty"
},
"realTimeRealization": {
- "name": "test_kylin_ii",
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii"
},
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 3c01e09..08a5a2e 100644
--- a/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -18,19 +18,14 @@
package org.apache.kylin.job.engine;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.tools.OptionsHelper;
+import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity;
+import java.io.*;
/**
* @author ysong1
@@ -62,14 +57,13 @@ public class JobEngineConfig {
File jobConfig = getJobConfig(hadoopJobConfFile);
if (jobConfig == null || !jobConfig.exists()) {
- logger.warn("fail to locate " + hadoopJobConfFile);
- }
- jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml");
- if (jobConfig == null || !jobConfig.exists()) {
- logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
- throw new RuntimeException("fail to locate " + hadoopJobConfFile);
+ logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
+ jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml");
+ if (jobConfig == null || !jobConfig.exists()) {
+ logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
+ throw new RuntimeException("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml");
+ }
}
-
return OptionsHelper.convertToFileURL(jobConfig.getAbsolutePath());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 84344ad..1997327 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -113,21 +113,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
return optionsHelper.hasOption(option);
}
- private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
-
protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException {
int retVal = 0;
long start = System.nanoTime();
- String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " to " + MAP_REDUCE_CLASSPATH);
- if (kylinHiveDependency != null) {
- final String classpath = job.getConfiguration().get(MAP_REDUCE_CLASSPATH);
- if (classpath == null) {
- job.getConfiguration().set(MAP_REDUCE_CLASSPATH, kylinHiveDependency);
- } else {
- job.getConfiguration().set(MAP_REDUCE_CLASSPATH, classpath + ":" + kylinHiveDependency);
- }
- }
if (isAsync) {
job.submit();
} else {
@@ -147,6 +135,33 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
System.exit(5);
}
}
+
+ private static final String MAP_REDUCE_CLASSPATH = "mapreduce.application.classpath";
+
+ protected void setJobClasspath(Job job) {
+ String jarPath = KylinConfig.getInstanceFromEnv().getKylinJobJarPath();
+ File jarFile = new File(jarPath);
+ if (jarFile.exists()) {
+ job.setJar(jarPath);
+ logger.info("append job jar: " + jarPath);
+ } else {
+ job.setJarByClass(this.getClass());
+ }
+
+ String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " to " + MAP_REDUCE_CLASSPATH);
+ if (kylinHiveDependency != null) {
+ // yarn classpath is comma separated
+ kylinHiveDependency = kylinHiveDependency.replace(":", ",");
+ Configuration jobConf = job.getConfiguration();
+ final String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null) {
+ jobConf.set(MAP_REDUCE_CLASSPATH, kylinHiveDependency);
+ } else {
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
+ }
+ }
+ }
public void addInputDirs(String input, Job job) throws IOException {
for (String inp : StringSplitter.split(input, ",")) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
index 773e62f..d5e5292 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityJob.java
@@ -49,20 +49,9 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
public static final String OUTPUT_PATH = "/tmp/cardinality";
- /**
- * This is the jar path
- */
- private String jarPath;
-
- private String table;
-
public HiveColumnCardinalityJob() {
}
- public HiveColumnCardinalityJob(String path, String tokenPath) {
- this.jarPath = path;
- }
-
@Override
public int run(String[] args) throws Exception {
@@ -80,19 +69,14 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
Configuration conf = getConf();
job = Job.getInstance(conf, jobName);
- // set job configuration - basic
- if (jarPath == null || !new File(jarPath).exists()) {
- job.setJarByClass(getClass());
- } else {
- job.setJar(jarPath);
- }
-
+ setJobClasspath(job);
+
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set("dfs.block.size", "67108864");
// Mapper
- this.table = getOptionValue(OPTION_TABLE);
+ String table = getOptionValue(OPTION_TABLE);
String[] dbTableNames = HadoopUtil.parseHiveTableName(table);
HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
index f0910df..a0d54cb 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CubeHFileJob.java
@@ -67,12 +67,7 @@ public class CubeHFileJob extends AbstractHadoopJob {
CubeInstance cube = cubeMgr.getCube(cubeName);
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
+ setJobClasspath(job);
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
FileOutputFormat.setOutputPath(job, output);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
index 0b3d272..be54ca7 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/CuboidJob.java
@@ -83,12 +83,7 @@ public class CuboidJob extends AbstractHadoopJob {
logger.info("Starting: " + job.getJobName());
FileInputFormat.setInputPaths(job, input);
- File jarFile = new File(config.getKylinJobJarPath());
- if (jarFile.exists()) {
- job.setJar(config.getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
+ setJobClasspath(job);
// Mapper
if (this.mapperClass == null) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
index 47c8c50..ccc2e0d 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/FactDistinctColumnsJob.java
@@ -18,7 +18,6 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.File;
import java.io.IOException;
import org.apache.commons.cli.Options;
@@ -31,15 +30,14 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author yangli9
@@ -71,6 +69,8 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
System.out.println("Starting: " + job.getJobName());
+ setJobClasspath(job);
+
setupMapper(intermediateTable);
setupReducer(output);
@@ -90,13 +90,6 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
private void setupMapper(String intermediateTable) throws IOException {
// FileInputFormat.setInputPaths(job, input);
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
-
String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
HCatInputFormat.setInput(job, dbTableNames[0],
dbTableNames[1]);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
index 7de432a..39efd8f 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/KeyDistributionJob.java
@@ -89,7 +89,7 @@ package org.apache.kylin.job.hadoop.cube;
// Job job = Job.getInstanceFromEnv(getConf(), jobName);
//
// // set job configuration - basic
-// job.setJarByClass(getClass());
+// setJobClasspath(job);
// addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
//
// Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
index b8b03be..0831d31 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MergeCuboidJob.java
@@ -61,13 +61,7 @@ public class MergeCuboidJob extends CuboidJob {
System.out.println("Starting: " + jobName);
job = Job.getInstance(getConf(), jobName);
- // set job configuration - basic
- File JarFile = new File(config.getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(config.getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
+ setJobClasspath(job);
// set inputs
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
index 8b77497..061cb90 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RangeKeyDistributionJob.java
@@ -68,12 +68,7 @@ public class RangeKeyDistributionJob extends AbstractHadoopJob {
String jobName = getOptionValue(OPTION_JOB_NAME);
job = Job.getInstance(getConf(), jobName);
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
+ setJobClasspath(job);
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
index a441006..faf6675 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/RowKeyDistributionCheckerJob.java
@@ -59,7 +59,7 @@ public class RowKeyDistributionCheckerJob extends AbstractHadoopJob {
String jobName = getOptionValue(OPTION_JOB_NAME);
job = Job.getInstance(getConf(), jobName);
- job.setJarByClass(this.getClass());
+ setJobClasspath(job);
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
index 4f2a568..c479b68 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
@@ -18,8 +18,6 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import java.io.File;
-
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -31,12 +29,11 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* @author yangli9
@@ -61,12 +58,7 @@ public class IICreateHFileJob extends AbstractHadoopJob {
job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME));
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
+ setJobClasspath(job);
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
FileOutputFormat.setOutputPath(job, output);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
index 5b67057..9c7051a 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIDistinctColumnsJob.java
@@ -18,7 +18,6 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import java.io.File;
import java.io.IOException;
import org.apache.commons.cli.Options;
@@ -31,16 +30,16 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.job.hadoop.hive.IIJoinedFlatTableDesc;
import org.apache.kylin.job.hadoop.hive.IntermediateColumnDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -74,6 +73,8 @@ public class IIDistinctColumnsJob extends AbstractHadoopJob {
job.getConfiguration().set(BatchConstants.TABLE_NAME, tableName);
job.getConfiguration().set(BatchConstants.TABLE_COLUMNS, getColumns(ii));
+ setJobClasspath(job);
+
setupMapper();
setupReducer(output);
@@ -99,13 +100,6 @@ public class IIDistinctColumnsJob extends AbstractHadoopJob {
private void setupMapper() throws IOException {
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
-
String tableName = job.getConfiguration().get(BatchConstants.TABLE_NAME);
String[] dbTableNames = HadoopUtil.parseHiveTableName(tableName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
index 547a499..6a0532e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/InvertedIndexJob.java
@@ -18,7 +18,6 @@
package org.apache.kylin.job.hadoop.invertedindex;
-import java.io.File;
import java.io.IOException;
import org.apache.commons.cli.Options;
@@ -31,15 +30,15 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
+import org.apache.kylin.job.constant.BatchConstants;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.job.constant.BatchConstants;
/**
* @author yangli9
@@ -69,6 +68,8 @@ public class InvertedIndexJob extends AbstractHadoopJob {
IIInstance ii = getII(iiname);
short sharding = ii.getDescriptor().getSharding();
+
+ setJobClasspath(job);
setupMapper(intermediateTable);
setupReducer(output, sharding);
@@ -103,13 +104,6 @@ public class InvertedIndexJob extends AbstractHadoopJob {
private void setupMapper(String intermediateTable) throws IOException {
- File JarFile = new File(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- if (JarFile.exists()) {
- job.setJar(KylinConfig.getInstanceFromEnv().getKylinJobJarPath());
- } else {
- job.setJarByClass(this.getClass());
- }
-
String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
HCatInputFormat.setInput(job, dbTableNames[0],
dbTableNames[1]);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
index c55e5d4..0f94d32 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/RandomKeyDistributionJob.java
@@ -63,7 +63,8 @@ public class RandomKeyDistributionJob extends AbstractHadoopJob {
String jobName = getOptionValue(OPTION_JOB_NAME);
job = Job.getInstance(getConf(), jobName);
- job.setJarByClass(this.getClass());
+ setJobClasspath(job);
+
addInputDirs(getOptionValue(OPTION_INPUT_PATH), job);
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/script/compress.sh
----------------------------------------------------------------------
diff --git a/script/compress.sh b/script/compress.sh
index 4dd91f3..f7d860a 100755
--- a/script/compress.sh
+++ b/script/compress.sh
@@ -16,6 +16,9 @@ echo 'package tgz'
rm -rf kylin-${version}
mkdir kylin-${version}
cp -r lib bin conf tomcat kylin-${version}
+find kylin-${version} -type d -exec chmod 755 {} \;
+find kylin-${version} -type f -exec chmod 644 {} \;
+find kylin-${version} -type f -name "*.sh" -exec chmod 755 {} \;
tar -cvzf kylin-${version}.tgz kylin-${version}
rm -rf kylin-${version}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
index 556467a..ce1a173 100755
--- a/script/package.sh
+++ b/script/package.sh
@@ -26,8 +26,8 @@ version=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpress
echo "kylin version: ${version}"
export version
-sh ${dir}/download-tomcat.sh || { exit 1; }
-sh ${dir}/build.sh || { exit 1; }
-sh ${dir}/prepare.sh || { exit 1; }
-sh ${dir}/compress.sh || { exit 1; }
+sh script/download-tomcat.sh || { exit 1; }
+sh script/build.sh || { exit 1; }
+sh script/prepare.sh || { exit 1; }
+sh script/compress.sh || { exit 1; }
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 657a912..b550fc0 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -14,7 +14,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.realization.SQLDigest;
import java.util.List;
-import java.util.UUID;
/**
* Created by shaoshi on 2/13/15.
@@ -42,7 +41,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
public void init() {
RealizationRegistry registry = RealizationRegistry.getInstance(config);
historyRealizationInstance = registry.getRealization(historyRealization.getType(), historyRealization.getRealization());
- realTimeRealizationInstance = registry.getRealization(realTimeRealizationInstance.getType(), historyRealization.getRealization());
+ realTimeRealizationInstance = registry.getRealization(realTimeRealization.getType(), realTimeRealization.getRealization());
}
@@ -131,4 +130,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
public IRealization getRealTimeRealizationInstance() {
return realTimeRealizationInstance;
}
+
+ @Override
+ public long getDateRangeStart() {
+ return historyRealizationInstance.getDateRangeStart();
+ }
+
+ @Override
+ public long getDateRangeEnd() {
+ return realTimeRealizationInstance.getDateRangeEnd();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index 20f5545..7e4621c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -98,8 +98,6 @@ public class HybridManager implements IRealizationProvider {
throw new IllegalStateException("HybridInstance must have both historyRealization and realTimeRealization set, at " + path);
}
- hybridInstance.init();
-
final String name = hybridInstance.getName();
hybridMap.putLocal(name, hybridInstance);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 5efa6e1..566a4f5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,7 +1,36 @@
package org.apache.kylin.storage.hybrid;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.storage.IStorageEngine;
+import org.apache.kylin.storage.StorageContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Created by shaoshi on 2/13/15.
*/
-public class HybridStorageEngine {
+public class HybridStorageEngine implements IStorageEngine {
+
+
+ private static final Logger logger = LoggerFactory.getLogger(HybridStorageEngine.class);
+
+ private HybridInstance hybridInstance;
+
+
+ public HybridStorageEngine(HybridInstance hybridInstance) {
+ this.hybridInstance = hybridInstance;
+
+ }
+
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest) {
+
+ long conditionBoundry = hybridInstance.getHistoryRealizationInstance().getDateRangeEnd();
+
+ TupleFilter filter = sqlDigest.filter;
+
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/37d3fe3d/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
index 1f1fd39..fbd18e2 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -38,7 +38,7 @@ public class HybridManagerTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() throws Exception {
HybridInstance cube = getHybridManager().getHybridInstance("test_kylin_hybrid");
-
+ cube.init();
System.out.println(JsonUtil.writeValueAsIndentString(cube));
IRealization history = cube.getHistoryRealizationInstance();