You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/10/27 00:30:28 UTC
[1/7] kylin git commit: KYLIN-2030 fix with KYLIN-1971
Repository: kylin
Updated Branches:
refs/heads/master ea3d02ff1 -> c804dc8d7
KYLIN-2030 fix with KYLIN-1971
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c804dc8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c804dc8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c804dc8d
Branch: refs/heads/master
Commit: c804dc8d77bb0402d6cd391e582a47f323edbb21
Parents: 324e423
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 14:41:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeCapabilityChecker.java | 13 +++----------
.../org/apache/kylin/metadata/model/ColumnDesc.java | 8 +-------
2 files changed, 4 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e509d98..a3c89e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,22 +25,18 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,11 +82,8 @@ public class CubeCapabilityChecker {
} else {
//for non query-on-facttable
if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
- TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
- Set<TblColRef> dimCols = Sets.newHashSet();
- for (ColumnDesc columnDesc : tableDesc.getColumns()) {
- dimCols.add(columnDesc.getRef());
- }
+
+ Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
//1. all aggregations on lookup table can be done. For distinct count, mark them all DimensionAsMeasures
// so that the measure has a chance to be upgraded to DimCountDistinctMeasureType in org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a13bd37..e0184b4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
/**
* Column Metadata from Source. All name should be uppercase.
@@ -203,11 +202,6 @@ public class ColumnDesc implements Serializable {
@Override
public String toString() {
- return "ColumnDesc{" +
- "id='" + id + '\'' +
- ", name='" + name + '\'' +
- ", datatype='" + datatype + '\'' +
- ", comment='" + comment + '\'' +
- '}';
+ return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';
}
}
[6/7] kylin git commit: KYLIN-2030 bug fix
Posted by ma...@apache.org.
KYLIN-2030 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/324e4239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/324e4239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/324e4239
Branch: refs/heads/master
Commit: 324e4239c308ba5b14a46eb3187dad3186489f92
Parents: 46284bd
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Oct 9 19:07:52 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 23 ++++++++++----------
.../kylin/query/relnode/OLAPAggregateRel.java | 22 +++++++++++--------
2 files changed, 25 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ee21b1c..e509d98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -81,7 +81,7 @@ public class CubeCapabilityChecker {
//1. dimension as measure
if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+ tryDimensionAsMeasures(unmatchedAggregations, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
}
} else {
//for non query-on-facttable
@@ -92,10 +92,18 @@ public class CubeCapabilityChecker {
dimCols.add(columnDesc.getRef());
}
- //1. dimension as measure, like max(cal_dt) or count( distinct col) from lookup
+ //1. all aggregations on lookup table can be done. For distinct count, mark them all DimensionAsMeasures
+ // so that the measure has a chance to be upgraded to DimCountDistinctMeasureType in org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols);
+ Iterator<FunctionDesc> itr = unmatchedAggregations.iterator();
+ while (itr.hasNext()) {
+ FunctionDesc functionDesc = itr.next();
+ if (dimCols.containsAll(functionDesc.getParameter().getColRefs())) {
+ itr.remove();
+ }
+ }
}
+ tryDimensionAsMeasures(Lists.newArrayList(aggrFunctions), result, dimCols);
//2. more "dimensions" contributed by snapshot
if (!unmatchedDimensions.isEmpty()) {
@@ -159,19 +167,12 @@ public class CubeCapabilityChecker {
return result;
}
- private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result, Set<TblColRef> dimCols) {
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
+ private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) {
Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
while (it.hasNext()) {
FunctionDesc functionDesc = it.next();
- if (cubeFuncs.contains(functionDesc)) {
- it.remove();
- continue;
- }
-
// let calcite handle count
if (functionDesc.isCount()) {
it.remove();
http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d73206a..62351d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -296,16 +296,20 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
private void translateAggregation() {
- // now the realization is known, replace aggregations with what's defined on MeasureDesc
- List<MeasureDesc> measures = this.context.realization.getMeasures();
- List<FunctionDesc> newAggrs = Lists.newArrayList();
- for (FunctionDesc aggFunc : this.aggregations) {
- newAggrs.add(findInMeasures(aggFunc, measures));
+ if (!noPrecaculatedFieldsAvailable()) {
+ // now the realization is known, replace aggregations with what's defined on MeasureDesc
+ List<MeasureDesc> measures = this.context.realization.getMeasures();
+ List<FunctionDesc> newAggrs = Lists.newArrayList();
+ for (FunctionDesc aggFunc : this.aggregations) {
+ newAggrs.add(findInMeasures(aggFunc, measures));
+ }
+ this.aggregations.clear();
+ this.aggregations.addAll(newAggrs);
+ this.context.aggregations.clear();
+ this.context.aggregations.addAll(newAggrs);
+ } else {
+ //the realization is not contributing pre-calculated fields at all
}
- this.aggregations.clear();
- this.aggregations.addAll(newAggrs);
- this.context.aggregations.clear();
- this.context.aggregations.addAll(newAggrs);
}
private FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) {
[2/7] kylin git commit: KYLIN-2030 enhancement
Posted by ma...@apache.org.
KYLIN-2030 enhancement
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc
Branch: refs/heads/master
Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e
Parents: 1429da5
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 43 +++++++++++++++-----
.../org/apache/kylin/cube/model/CubeDesc.java | 6 +--
.../kylin/metadata/model/FunctionDesc.java | 1 +
.../test/resources/query/sql_lookup/query06.sql | 1 +
.../test/resources/query/sql_lookup/query07.sql | 1 +
5 files changed, 39 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube);
// try custom measure types
+ // in RAW query, unmatchedDimensions and unmatchedAggregations will null, so can't chose RAW cube well!
+ // if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) {
tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
+ // }
- // try dimension-as-measure
- if (!unmatchedAggregations.isEmpty()) {
- if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+ //more tricks
+ if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+ //for query-on-facttable
+ //1. dimension as measure
+
+ if (!unmatchedAggregations.isEmpty()) {
tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
- } else {
- //deal with query on lookup table, like https://issues.apache.org/jira/browse/KYLIN-2030
- if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
- Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+ }
+ } else {
+ //for non query-on-facttable
+ if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+ TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+ Set<TblColRef> dimCols = Sets.newHashSet();
+ for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+ dimCols.add(columnDesc.getRef());
+ }
+
+ //1. dimension as measure, like max(cal_dt) or count( distinct col) from lookup
+ if (!unmatchedAggregations.isEmpty()) {
tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols);
- } else {
- logger.info("Skip tryDimensionAsMeasures because current cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable);
}
+
+ //2. more "dimensions" contributed by snapshot
+ if (!unmatchedDimensions.isEmpty()) {
+ unmatchedDimensions.removeAll(dimCols);
+ }
+ } else {
+ logger.info("cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0c0b4f3..159042d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -33,9 +33,9 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
+import java.util.Map.Entry;
import javax.annotation.Nullable;
@@ -67,9 +67,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
@@ -674,7 +674,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private boolean hasSingle(ArrayList<Set<String>> dimsList) {
boolean hasSingle = false;
for (Set<String> dims : dimsList) {
- if (dims.size() < 2)
+ if (dims.size() == 1)
hasSingle = true;
}
return hasSingle;
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index e7d5186..ae7f805 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -102,6 +102,7 @@ public class FunctionDesc {
}
public MeasureType<?> getMeasureType() {
+ //like max(cal_dt)
if (isDimensionAsMetric && !isCountDistinct()) {
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query06.sql b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
new file mode 100644
index 0000000..e2b8469
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
@@ -0,0 +1 @@
+select max(cal_dt) as x from edw.test_cal_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query07.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query07.sql b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
new file mode 100644
index 0000000..6be3439
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
@@ -0,0 +1 @@
+select site_id,count(*) as y,count(DISTINCT site_name) as x from edw.test_sites group by site_id
[5/7] kylin git commit: port KYLIN-2012 to new interface introduced
in KYLIN-2125
Posted by ma...@apache.org.
port KYLIN-2012 to new interface introduced in KYLIN-2125
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/615e21d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/615e21d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/615e21d7
Branch: refs/heads/master
Commit: 615e21d7fb0dae651ab643949ef5078bef7b12ea
Parents: ca6837d
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 5 ++-
.../kylin/source/hive/BeelineHiveClient.java | 2 +-
.../source/hive/HiveSourceTableLoader.java | 32 +++++++--------
.../apache/kylin/source/hive/SchemaChecker.java | 41 ++++++++------------
4 files changed, 36 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
import org.apache.kylin.source.kafka.TimedJsonStreamParser;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
String tableFileDir = temp.getParent();
temp.delete();
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
//BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
- HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test001");
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part");
System.out.println(hiveTableMeta);
loader.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
/**
@@ -49,27 +52,25 @@ public class HiveSourceTableLoader {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
- public static final String OUTPUT_SURFIX = "json";
- public static final String TABLE_FOLDER_NAME = "table";
- public static final String TABLE_EXD_FOLDER_NAME = "table_exd";
-
public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
- Map<String, Set<String>> db2tables = Maps.newHashMap();
- for (String table : hiveTables) {
- String[] parts = HadoopUtil.parseHiveTableName(table);
- Set<String> set = db2tables.get(parts[0]);
- if (set == null) {
- set = Sets.newHashSet();
- db2tables.put(parts[0], set);
- }
- set.add(parts[1]);
+ SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
+ for (String fullTableName : hiveTables) {
+ String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
+ db2tables.put(parts[0], parts[1]);
+ }
+
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config));
+ for (Map.Entry<String, String> entry : db2tables.entries()) {
+ SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue());
+ result.raiseExceptionWhenInvalid();
}
// extract from hive
Set<String> loadedTables = Sets.newHashSet();
for (String database : db2tables.keySet()) {
- List<String> loaded = extractHiveTables(database, db2tables.get(database), config);
+ List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient);
loadedTables.addAll(loaded);
}
@@ -82,12 +83,11 @@ public class HiveSourceTableLoader {
metaMgr.removeTableExd(hiveTable);
}
- private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException {
+ private static List<String> extractHiveTables(String database, Set<String> tables, IHiveClient hiveClient) throws IOException {
List<String> loadedTables = Lists.newArrayList();
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
for (String tableName : tables) {
- IHiveClient hiveClient = HiveClientFactory.getHiveClient();
HiveTableMeta hiveTableMeta;
try {
hiveTableMeta = hiveClient.getHiveTableMeta(database, tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index 319ebee..87a8870 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -27,8 +27,6 @@ import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -46,7 +44,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class SchemaChecker {
- private final HiveClient hiveClient;
+ private final IHiveClient hiveClient;
private final MetadataManager metadataManager;
private final CubeManager cubeManager;
@@ -87,23 +85,16 @@ public class SchemaChecker {
}
}
- SchemaChecker(HiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) {
+ SchemaChecker(IHiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) {
this.hiveClient = checkNotNull(hiveClient, "hiveClient is null");
this.metadataManager = checkNotNull(metadataManager, "metadataManager is null");
this.cubeManager = checkNotNull(cubeManager, "cubeManager is null");
}
- private List<FieldSchema> fetchSchema(String dbName, String tblName) throws Exception {
- List<FieldSchema> fields = Lists.newArrayList();
- fields.addAll(hiveClient.getHiveTableFields(dbName, tblName));
-
- Table table = hiveClient.getHiveTable(dbName, tblName);
- List<FieldSchema> partitionFields = table.getPartitionKeys();
- if (partitionFields != null) {
- fields.addAll(partitionFields);
- }
-
- return fields;
+ private List<HiveTableMeta.HiveTableColumnMeta> fetchSchema(String dbName, String tblName) throws Exception {
+ List<HiveTableMeta.HiveTableColumnMeta> columnMetas = Lists.newArrayList();
+ columnMetas.addAll(hiveClient.getHiveTableMeta(dbName, tblName).allColumns);
+ return columnMetas;
}
private List<CubeInstance> findCubeByTable(final String fullTableName) {
@@ -128,12 +119,12 @@ public class SchemaChecker {
return ImmutableList.copyOf(relatedCubes);
}
- private boolean isColumnCompatible(ColumnDesc column, FieldSchema field) {
- if (!column.getName().equalsIgnoreCase(field.getName())) {
+ private boolean isColumnCompatible(ColumnDesc column, HiveTableMeta.HiveTableColumnMeta field) {
+ if (!column.getName().equalsIgnoreCase(field.name)) {
return false;
}
- String typeStr = field.getType();
+ String typeStr = field.dataType;
// kylin uses double internally for float, see HiveSourceTableLoader.java
// TODO should this normalization to be in DataType class ?
if ("float".equalsIgnoreCase(typeStr)) {
@@ -159,7 +150,7 @@ public class SchemaChecker {
* @param fieldsMap current hive schema of `table`
* @return true if all columns used in `cube` has compatible schema with `fieldsMap`, false otherwise
*/
- private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
+ private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, HiveTableMeta.HiveTableColumnMeta> fieldsMap) {
Set<ColumnDesc> usedColumns = Sets.newHashSet();
for (TblColRef col : cube.getAllColumns()) {
usedColumns.add(col.getColumnDesc());
@@ -168,7 +159,7 @@ public class SchemaChecker {
List<String> violateColumns = Lists.newArrayList();
for (ColumnDesc column : table.getColumns()) {
if (usedColumns.contains(column)) {
- FieldSchema field = fieldsMap.get(column.getName());
+ HiveTableMeta.HiveTableColumnMeta field = fieldsMap.get(column.getName());
if (field == null || !isColumnCompatible(column, field)) {
violateColumns.add(column.getName());
}
@@ -184,7 +175,7 @@ public class SchemaChecker {
* @param fields current table metadata in hive
* @return true if only new columns are appended in hive, false otherwise
*/
- private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields) {
+ private boolean checkAllColumnsInTableDesc(TableDesc table, List<HiveTableMeta.HiveTableColumnMeta> fields) {
if (table.getColumnCount() > fields.size()) {
return false;
}
@@ -206,15 +197,15 @@ public class SchemaChecker {
return CheckResult.validOnFirstLoad(fullTableName);
}
- List<FieldSchema> currentFields;
- Map<String, FieldSchema> currentFieldsMap = Maps.newHashMap();
+ List<HiveTableMeta.HiveTableColumnMeta> currentFields;
+ Map<String, HiveTableMeta.HiveTableColumnMeta> currentFieldsMap = Maps.newHashMap();
try {
currentFields = fetchSchema(dbName, tblName);
} catch (Exception e) {
return CheckResult.invalidOnFetchSchema(fullTableName, e);
}
- for (FieldSchema field : currentFields) {
- currentFieldsMap.put(field.getName().toUpperCase(), field);
+ for (HiveTableMeta.HiveTableColumnMeta field : currentFields) {
+ currentFieldsMap.put(field.name.toUpperCase(), field);
}
List<String> issues = Lists.newArrayList();
[7/7] kylin git commit: improve integer type upgrade logic
Posted by ma...@apache.org.
improve integer type upgrade logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1429da5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1429da5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1429da5f
Branch: refs/heads/master
Commit: 1429da5f650852402d1494785368e9175f13c379
Parents: 615e21d
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 30 18:31:04 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../filter/EvaluatableFunctionTupleFilter.java | 151 -------------------
.../metadata/filter/TupleFilterSerializer.java | 20 ++-
.../apache/kylin/metadata/model/ColumnDesc.java | 20 ++-
.../apache/kylin/metadata/tuple/TupleInfo.java | 4 +-
.../apache/kylin/query/schema/OLAPTable.java | 31 +++-
5 files changed, 63 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
deleted file mode 100644
index ff24172..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-import com.google.common.collect.Lists;
-
-public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter {
-
- private boolean constantsInitted = false;
-
- //about non-like
- private List<Object> values;
- private Object tupleValue;
-
- public EvaluatableFunctionTupleFilter(String name) {
- super(name, FilterOperatorEnum.EVAL_FUNC);
- values = Lists.newArrayListWithCapacity(1);
- values.add(null);
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
- // extract tuple value
- Object tupleValue = null;
- for (TupleFilter filter : this.children) {
- if (!isConstant(filter)) {
- filter.evaluate(tuple, cs);
- tupleValue = filter.getValues().iterator().next();
- }
- }
-
- TblColRef tblColRef = this.getColumn();
- DataType strDataType = DataType.getType("string");
- if (tblColRef.getType() != strDataType) {
- throw new IllegalStateException("Only String type is allow in BuiltInFunction");
- }
- ByteArray valueByteArray = (ByteArray) tupleValue;
- StringSerializer serializer = new StringSerializer(strDataType);
- String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
-
- try {
- if (isLikeFunction()) {
- return (Boolean) invokeFunction(value);
- } else {
- this.tupleValue = invokeFunction(value);
- //convert back to ByteArray format because the outer EvaluatableFunctionTupleFilter assumes input as ByteArray
- ByteBuffer buffer = ByteBuffer.allocate(valueByteArray.length() * 2);
- serializer.serialize((String) this.tupleValue, buffer);
- this.tupleValue = new ByteArray(buffer.array(), 0, buffer.position());
-
- return true;
- }
- } catch (InvocationTargetException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Collection<?> getValues() {
- this.values.set(0, tupleValue);
- return values;
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- if (!isValid()) {
- throw new IllegalStateException("must be valid");
- }
- BytesUtil.writeUTFString(name, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- private boolean isConstant(TupleFilter filter) {
- return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
- }
-
- @Override
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (isLikeFunction())
- initConstants();
- return super.invokeFunction(input);
- }
-
- private void initConstants() {
- if (constantsInitted) {
- return;
- }
- //will replace the ByteArray pattern to String type
- ByteArray byteArray = (ByteArray) methodParams.get(constantPosition);
- StringSerializer s = new StringSerializer(DataType.getType("string"));
- String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
- //TODO
- //pattern = pattern.toLowerCase();//to remove upper case
- methodParams.set(constantPosition, pattern);
- constantsInitted = true;
- }
-
- //even for "tolower(s)/toupper(s)/substring(like) like pattern", the like pattern can be used for index searching
- public String getLikePattern() {
- if (!isLikeFunction()) {
- return null;
- }
-
- initConstants();
- return (String) methodParams.get(1);
- }
-
- public boolean isLikeFunction() {
- return "like".equalsIgnoreCase(this.getName());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index a051ea9..2df474e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.lang.reflect.InvocationTargetException;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -29,6 +30,8 @@ import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
+
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
*
@@ -45,6 +48,7 @@ public class TupleFilterSerializer {
private static final int BUFFER_SIZE = 65536;
private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>();
+ protected static final Map<TupleFilter.FilterOperatorEnum, Class> extendedTupleFilters = Maps.newHashMap();
static {
for (TupleFilter.FilterOperatorEnum op : TupleFilter.FilterOperatorEnum.values()) {
@@ -191,14 +195,20 @@ public class TupleFilterSerializer {
case UNSUPPORTED:
filter = new UnsupportedTupleFilter(op);
break;
- case EVAL_FUNC:
- filter = new EvaluatableFunctionTupleFilter(null);
- break;
case MASSIN:
filter = new MassInTupleFilter();
break;
- default:
- throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+ default: {
+ if (extendedTupleFilters.containsKey(op)) {
+ try {
+ filter = (TupleFilter) extendedTupleFilters.get(op).getConstructor().newInstance();
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+ }
+ }
}
return filter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 3bf0de9..a13bd37 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
/**
* Column Metadata from Source. All name should be uppercase.
@@ -47,6 +48,7 @@ public class ColumnDesc implements Serializable {
// parsed from data type
private DataType type;
+ private DataType upgradedType;
private TableDesc table;
private int zeroBasedIndex = -1;
@@ -80,6 +82,18 @@ public class ColumnDesc implements Serializable {
type = DataType.getType(datatype);
}
+ public void setUpgradedType(String datatype) {
+ this.upgradedType = DataType.getType(datatype);
+ }
+
+ public DataType getUpgradedType() {
+ if (this.upgradedType == null) {
+ return this.type;
+ } else {
+ return this.upgradedType;
+ }
+ }
+
public String getId() {
return id;
}
@@ -171,19 +185,19 @@ public class ColumnDesc implements Serializable {
if (getClass() != obj.getClass())
return false;
ColumnDesc other = (ColumnDesc) obj;
-
+
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
-
+
if (datatype == null) {
if (other.datatype != null)
return false;
} else if (!datatype.equals(other.datatype))
return false;
-
+
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
index f7c3b57..8970124 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
@@ -94,9 +94,9 @@ public class TupleInfo {
columns.add(index, col);
if (dataTypeNames.size() > index)
- dataTypeNames.set(index, col.getType().getName());
+ dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName());
else
- dataTypeNames.add(index, col.getType().getName());
+ dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName());
}
public List<String> getAllFields() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 4994433..ac70716 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -43,15 +43,21 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.enumerator.OLAPQuery.EnumeratorTypeEnum;
import org.apache.kylin.query.relnode.OLAPTableScan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -61,6 +67,8 @@ import com.google.common.collect.Sets;
*/
public class OLAPTable extends AbstractQueryableTable implements TranslatableTable {
+ protected static final Logger logger = LoggerFactory.getLogger(OLAPTable.class);
+
private static Map<String, SqlTypeName> SQLTYPE_MAPPING = new HashMap<String, SqlTypeName>();
static {
@@ -123,7 +131,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : exposedColumns) {
- RelDataType sqlType = createSqlType(typeFactory, column.getType(), column.isNullable());
+ RelDataType sqlType = createSqlType(typeFactory, column.getUpgradedType(), column.isNullable());
sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory);
fieldInfo.add(column.getName(), sqlType);
}
@@ -176,6 +184,8 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
//if exist sum(x), where x is integer/short/byte
//to avoid overflow we upgrade x's type to long
+ //this includes checking two parts:
+ //1. sum measures in cubes:
HashSet<ColumnDesc> updateColumns = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
@@ -187,9 +197,26 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
}
}
}
+ //2. All integer measures in non-cube realizations
+ MetadataManager metadataManager = MetadataManager.getInstance(olapSchema.getConfig());
+ for (IRealization realization : mgr.listAllRealizations(olapSchema.getProjectName())) {
+ if (realization.getType() == RealizationType.INVERTED_INDEX && sourceTable.getIdentity().equalsIgnoreCase(realization.getFactTable())) {
+ DataModelDesc dataModelDesc = realization.getDataModelDesc();
+ for (String metricColumn : dataModelDesc.getMetrics()) {
+ ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getFactTable() + "." + metricColumn);
+ if (columnDesc.getType().isIntegerFamily() && !columnDesc.getType().isBigInt())
+ updateColumns.add(columnDesc);
+ }
+ }
+ }
+
for (ColumnDesc upgrade : updateColumns) {
int index = tableColumns.indexOf(upgrade);
- tableColumns.get(index).setDatatype("bigint");
+ if (index < 0) {
+ throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");
+ }
+ tableColumns.get(index).setUpgradedType("bigint");
+ logger.info("To avoid overflow, upgraded {}'s type from {} to {}", tableColumns.get(index), tableColumns.get(index).getType(), tableColumns.get(index).getUpgradedType());
}
return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
[3/7] kylin git commit: port KYLIN-2068 to
BeelineHiveClient(KYLIN-2125)
Posted by ma...@apache.org.
port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca6837d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca6837d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca6837d0
Branch: refs/heads/master
Commit: ca6837d004c0da31c95027b48db589238179f1ed
Parents: e2a932e
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/BeelineHiveClient.java | 10 +++++-----
.../java/org/apache/kylin/source/hive/CLIHiveClient.java | 4 ++--
.../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 +
.../java/org/apache/kylin/source/hive/HiveTableMeta.java | 6 ++++--
4 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
while (columns.next()) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12)));
}
builder.setAllColumns(allColumns);
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
if ("".equals(resultSet.getString(1).trim())) {
break;
}
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim()));
}
builder.setPartitionColumns(partitionColumns);
}
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
public static void main(String[] args) throws SQLException {
- //BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
- BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
- HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "events");
+ BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ //BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test001");
System.out.println(hiveTableMeta);
loader.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
for (FieldSchema fieldSchema : allFields) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), fieldSchema.getComment()));
}
if (partitionFields != null && partitionFields.size() > 0) {
for (FieldSchema fieldSchema : partitionFields) {
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), fieldSchema.getComment()));
}
}
builder.setAllColumns(allColumns);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 346d278..388e72b 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -120,6 +120,7 @@ public class HiveSourceTableLoader {
cdesc.setDatatype(field.dataType);
}
cdesc.setId(String.valueOf(i + 1));
+ cdesc.setComment(field.comment);
columns.add(cdesc);
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
index c2b7c96..784a0bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -24,15 +24,17 @@ class HiveTableMeta {
static class HiveTableColumnMeta {
String name;
String dataType;
+ String comment;
- public HiveTableColumnMeta(String name, String dataType) {
+ public HiveTableColumnMeta(String name, String dataType, String comment) {
this.name = name;
this.dataType = dataType;
+ this.comment = comment;
}
@Override
public String toString() {
- return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + '}';
+ return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + ", comment='" + comment + '\'' + '}';
}
}
[4/7] kylin git commit: KYLIN-2125 add BeelineHiveClient
Posted by ma...@apache.org.
KYLIN-2125 add BeelineHiveClient
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2a932e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2a932e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2a932e7
Branch: refs/heads/master
Commit: e2a932e7439d0034b26b99897d7c293cadb1b8e3
Parents: ea3d02f
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/TableController.java | 9 +-
source-hive/pom.xml | 5 +
.../kylin/source/hive/BeelineHiveClient.java | 214 +++++++++++++++++++
.../source/hive/BeelineOptionsProcessor.java | 47 ++++
.../apache/kylin/source/hive/CLIHiveClient.java | 169 +++++++++++++++
.../apache/kylin/source/hive/HiveClient.java | 170 ---------------
.../kylin/source/hive/HiveClientFactory.java | 33 +++
.../source/hive/HiveSourceTableLoader.java | 89 ++++----
.../org/apache/kylin/source/hive/HiveTable.java | 16 +-
.../apache/kylin/source/hive/HiveTableMeta.java | 71 ++++++
.../kylin/source/hive/HiveTableMetaBuilder.java | 102 +++++++++
.../apache/kylin/source/hive/HqlExecutable.java | 107 ----------
.../apache/kylin/source/hive/IHiveClient.java | 36 ++++
.../hive/BeelineOptionsProcessorTest.java | 38 ++++
14 files changed, 769 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
tableName = dbTableName[0] + "." + dbTableName[1];
TableDesc desc = cubeMgmtService.getMetadataManager().getTableDesc(tableName);
- if(desc == null)
+ if (desc == null)
return false;
tableType = desc.getSourceType();
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive", method = { RequestMethod.GET })
@ResponseBody
private static List<String> showHiveDatabases() throws IOException {
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
List<String> results = null;
try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
@ResponseBody
private static List<String> showHiveTables(@PathVariable String database) throws IOException {
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
List<String> results = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<classifier>hadoop2</classifier>
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 0000000..0fbc39b
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class BeelineHiveClient implements IHiveClient {
+
+ private Connection cnct;
+ private Statement stmt;
+ private DatabaseMetaData metaData;
+
+ public BeelineHiveClient(String beelineParams) {
+ if (StringUtils.isEmpty(beelineParams)) {
+ throw new IllegalArgumentException("BeelineParames cannot be empty");
+ }
+ String[] splits = StringUtils.split(beelineParams);
+ String url = null, username = null, password = null;
+ for (int i = 0; i < splits.length; i++) {
+ if ("-u".equals(splits[i])) {
+ url = stripQuotes(splits[i + 1]);
+ }
+ if ("-n".equals(splits[i])) {
+ username = stripQuotes(splits[i + 1]);
+ }
+ if ("-p".equals(splits[i])) {
+ password = stripQuotes(splits[i + 1]);
+ }
+ }
+ this.init(url, username, password);
+ }
+
+ private void init(String url, String username, String password) {
+ try {
+ Class.forName("org.apache.hive.jdbc.HiveDriver");
+ cnct = DriverManager.getConnection(url, username, password);
+ stmt = cnct.createStatement();
+ metaData = cnct.getMetaData();
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String stripQuotes(String input) {
+ if (input.startsWith("'") && input.endsWith("'")) {
+ return StringUtils.strip(input, "'");
+ } else if (input.startsWith("\"") && input.endsWith("\"")) {
+ return StringUtils.strip(input, "\"");
+ } else {
+ return input;
+ }
+ }
+
+ public List<String> getHiveDbNames() throws Exception {
+ List<String> ret = Lists.newArrayList();
+ ResultSet schemas = metaData.getSchemas();
+ while (schemas.next()) {
+ ret.add(String.valueOf(schemas.getObject(1)));
+ }
+ return ret;
+ }
+
+ public List<String> getHiveTableNames(String database) throws Exception {
+ List<String> ret = Lists.newArrayList();
+ ResultSet tables = metaData.getTables(null, database, null, null);
+ while (tables.next()) {
+ ret.add(String.valueOf(tables.getObject(3)));
+ }
+ return ret;
+ }
+
+ @Override
+ public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws SQLException {
+ ResultSet columns = metaData.getColumns(null, database, tableName, null);
+ HiveTableMetaBuilder builder = new HiveTableMetaBuilder();
+ builder.setTableName(tableName);
+
+ List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ while (columns.next()) {
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+ }
+ builder.setAllColumns(allColumns);
+
+ stmt.execute("use " + database);
+ ResultSet resultSet = stmt.executeQuery("describe formatted " + tableName);
+ extractHiveTableMeta(resultSet, builder);
+
+ return builder.createHiveTableMeta();
+ }
+
+ private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
+ while (resultSet.next()) {
+
+ List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+ if ("# Partition Information".equals(resultSet.getString(1).trim())) {
+ resultSet.next();
+ Preconditions.checkArgument("# col_name".equals(resultSet.getString(1).trim()));
+ resultSet.next();
+ Preconditions.checkArgument("".equals(resultSet.getString(1).trim()));
+ while (resultSet.next()) {
+ if ("".equals(resultSet.getString(1).trim())) {
+ break;
+ }
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim()));
+ }
+ builder.setPartitionColumns(partitionColumns);
+ }
+
+ if ("Owner:".equals(resultSet.getString(1).trim())) {
+ builder.setOwner(resultSet.getString(2).trim());
+ }
+ if ("LastAccessTime:".equals(resultSet.getString(1).trim())) {
+ try {
+ int i = Integer.parseInt(resultSet.getString(2).trim());
+ builder.setLastAccessTime(i);
+ } catch (NumberFormatException e) {
+ builder.setLastAccessTime(0);
+ }
+ }
+ if ("Location:".equals(resultSet.getString(1).trim())) {
+ builder.setSdLocation(resultSet.getString(2).trim());
+ }
+ if ("Table Type:".equals(resultSet.getString(1).trim())) {
+ builder.setTableType(resultSet.getString(2).trim());
+ }
+ if ("Table Parameters:".equals(resultSet.getString(1).trim())) {
+ while (resultSet.next()) {
+ if (resultSet.getString(2) == null) {
+ break;
+ }
+ if ("storage_handler".equals(resultSet.getString(2).trim())) {
+ builder.setIsNative(false);//default is true
+ }
+ if ("totalSize".equals(resultSet.getString(2).trim())) {
+ builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false
+ }
+ if ("numFiles".equals(resultSet.getString(2).trim())) {
+ builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
+ }
+ }
+ }
+ if ("InputFormat:".equals(resultSet.getString(1).trim())) {
+ builder.setSdInputFormat(resultSet.getString(2).trim());
+ }
+ if ("OutputFormat:".equals(resultSet.getString(1).trim())) {
+ builder.setSdOutputFormat(resultSet.getString(2).trim());
+ }
+ }
+ }
+
+ public void close() {
+ if (this.stmt != null) {
+ try {
+ this.stmt.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ if (this.cnct != null) {
+ try {
+ this.cnct.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ }
+
+ public static void main(String[] args) throws SQLException {
+
+ //BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "events");
+ System.out.println(hiveTableMeta);
+ loader.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
new file mode 100644
index 0000000..68cb352
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+public class BeelineOptionsProcessor {
+ private final Options options = new Options();
+
+ public BeelineOptionsProcessor() {
+
+ options.addOption(OptionBuilder.hasArg().withArgName("url").create('u'));
+ options.addOption(OptionBuilder.hasArg().withArgName("username").create('n'));
+ options.addOption(OptionBuilder.hasArg().withArgName("password").create('p'));
+
+ }
+
+ public CommandLine process(String[] argv) {
+ try {
+ return new GnuParser().parse(options, argv);
+
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
new file mode 100644
index 0000000..ea74470
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.hive;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Hive meta API client for Kylin
+ * @author shaoshi
+ *
+ */
+public class CLIHiveClient implements IHiveClient {
+
+ protected HiveConf hiveConf = null;
+ protected Driver driver = null;
+ protected HiveMetaStoreClient metaStoreClient = null;
+
+ public CLIHiveClient() {
+ hiveConf = new HiveConf(CLIHiveClient.class);
+ }
+
+ /**
+ * only used by Deploy Util
+ */
+ @Override
+ public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ CommandProcessorResponse response = getDriver().run(hql);
+ int retCode = response.getResponseCode();
+ if (retCode != 0) {
+ String err = response.getErrorMessage();
+ throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
+ }
+ }
+
+ /**
+ * only used by Deploy Util
+ */
+ @Override
+ public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ for (String sql : hqls)
+ executeHQL(sql);
+ }
+
+ @Override
+ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception {
+ HiveTableMetaBuilder builder = new HiveTableMetaBuilder();
+ Table table = getMetaStoreClient().getTable(database, tableName);
+
+ List<FieldSchema> allFields = getMetaStoreClient().getFields(database, tableName);
+ List<FieldSchema> partitionFields = table.getPartitionKeys();
+ if (allFields == null) {
+ allFields = Lists.newArrayList();
+ }
+ if (partitionFields != null && partitionFields.size() > 0) {
+ allFields.addAll(partitionFields);
+ }
+ List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+ for (FieldSchema fieldSchema : allFields) {
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ }
+ if (partitionFields != null && partitionFields.size() > 0) {
+ for (FieldSchema fieldSchema : partitionFields) {
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ }
+ }
+ builder.setAllColumns(allColumns);
+ builder.setPartitionColumns(partitionColumns);
+
+ builder.setSdLocation(table.getSd().getLocation());
+ builder.setFileSize(getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE));
+ builder.setFileNum(getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES));
+ builder.setIsNative(!MetaStoreUtils.isNonNativeTable(table));
+ builder.setTableName(tableName);
+ builder.setSdInputFormat(table.getSd().getInputFormat());
+ builder.setSdOutputFormat(table.getSd().getOutputFormat());
+ builder.setOwner(table.getOwner());
+ builder.setLastAccessTime(table.getLastAccessTime());
+ builder.setTableType(table.getTableType());
+
+ return builder.createHiveTableMeta();
+ }
+
+ @Override
+ public List<String> getHiveDbNames() throws Exception {
+ return getMetaStoreClient().getAllDatabases();
+ }
+
+ @Override
+ public List<String> getHiveTableNames(String database) throws Exception {
+ return getMetaStoreClient().getAllTables(database);
+ }
+
+ private HiveMetaStoreClient getMetaStoreClient() throws Exception {
+ if (metaStoreClient == null) {
+ metaStoreClient = new HiveMetaStoreClient(hiveConf);
+ }
+ return metaStoreClient;
+ }
+
+ /**
+ * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
+ *
+ * Get basic stats of table
+ * @param table
+ * - table
+ * @param statType
+ * - type of stats
+ * @return value of stats
+ */
+ private long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
+ Map<String, String> params = table.getParameters();
+ long result = 0;
+
+ if (params != null) {
+ try {
+ result = Long.parseLong(params.get(statType));
+ } catch (NumberFormatException e) {
+ result = 0;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get the hive ql driver to execute ddl or dml
+ * @return
+ */
+ private Driver getDriver() {
+ if (driver == null) {
+ driver = new Driver(hiveConf);
+ SessionState.start(new CliSessionState(hiveConf));
+ }
+
+ return driver;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
deleted file mode 100644
index a99b304..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-/**
- * Hive meta API client for Kylin
- * @author shaoshi
- *
- */
-public class HiveClient {
-
- protected HiveConf hiveConf = null;
- protected Driver driver = null;
- protected HiveMetaStoreClient metaStoreClient = null;
-
- public HiveClient() {
- hiveConf = new HiveConf(HiveClient.class);
- }
-
- public HiveClient(Map<String, String> configMap) {
- this();
- appendConfiguration(configMap);
- }
-
- public HiveConf getHiveConf() {
- return hiveConf;
- }
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
-
- /**
- * Append or overwrite the default hive client configuration; You need call this before invoke #executeHQL;
- * @param configMap
- */
- public void appendConfiguration(Map<String, String> configMap) {
- if (configMap != null && configMap.size() > 0) {
- for (Entry<String, String> e : configMap.entrySet()) {
- hiveConf.set(e.getKey(), e.getValue());
- }
- }
- }
-
- /**
- *
- * @param hql
- * @throws CommandNeedRetryException
- * @throws IOException
- */
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
- }
- }
-
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
- for (String sql : hqls)
- executeHQL(sql);
- }
-
- private HiveMetaStoreClient getMetaStoreClient() throws Exception {
- if (metaStoreClient == null) {
- metaStoreClient = new HiveMetaStoreClient(hiveConf);
- }
- return metaStoreClient;
- }
-
- public Table getHiveTable(String database, String tableName) throws Exception {
- return getMetaStoreClient().getTable(database, tableName);
- }
-
- public List<FieldSchema> getHiveTableFields(String database, String tableName) throws Exception {
- return getMetaStoreClient().getFields(database, tableName);
- }
-
- public String getHiveTableLocation(String database, String tableName) throws Exception {
- Table t = getHiveTable(database, tableName);
- return t.getSd().getLocation();
- }
-
- public long getFileSizeForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE);
- }
-
- public long getFileNumberForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
- }
-
- public List<String> getHiveDbNames() throws Exception {
- return getMetaStoreClient().getAllDatabases();
- }
-
- public List<String> getHiveTableNames(String database) throws Exception {
- return getMetaStoreClient().getAllTables(database);
- }
-
- /**
- * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
- *
- * Get basic stats of table
- * @param table
- * - table
- * @param statType
- * - type of stats
- * @return value of stats
- */
- public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
- Map<String, String> params = table.getParameters();
- long result = 0;
-
- if (params != null) {
- try {
- result = Long.parseLong(params.get(statType));
- } catch (NumberFormatException e) {
- result = 0;
- }
- }
- return result;
- }
-
- public boolean isNativeTable(String database, String tableName) throws Exception {
- return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
new file mode 100644
index 0000000..8c883af
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.kylin.common.KylinConfig;
+
+public class HiveClientFactory {
+ public static IHiveClient getHiveClient() {
+ if ("cli".equals(KylinConfig.getInstanceFromEnv().getHiveClientMode())) {
+ return new CLIHiveClient();
+ } else if ("beeline".equals(KylinConfig.getInstanceFromEnv().getHiveClientMode())) {
+ return new BeelineHiveClient(KylinConfig.getInstanceFromEnv().getHiveBeelineParams());
+ } else {
+ throw new RuntimeException("cannot recognize hive client mode");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index ec9aedb..346d278 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -25,10 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
@@ -37,10 +34,8 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
/**
@@ -54,25 +49,27 @@ public class HiveSourceTableLoader {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
- public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
+ public static final String OUTPUT_SURFIX = "json";
+ public static final String TABLE_FOLDER_NAME = "table";
+ public static final String TABLE_EXD_FOLDER_NAME = "table_exd";
- SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
- for (String fullTableName : hiveTables) {
- String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
- db2tables.put(parts[0], parts[1]);
- }
+ public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
- HiveClient hiveClient = new HiveClient();
- SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config));
- for (Map.Entry<String, String> entry : db2tables.entries()) {
- SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue());
- result.raiseExceptionWhenInvalid();
+ Map<String, Set<String>> db2tables = Maps.newHashMap();
+ for (String table : hiveTables) {
+ String[] parts = HadoopUtil.parseHiveTableName(table);
+ Set<String> set = db2tables.get(parts[0]);
+ if (set == null) {
+ set = Sets.newHashSet();
+ db2tables.put(parts[0], set);
+ }
+ set.add(parts[1]);
}
// extract from hive
Set<String> loadedTables = Sets.newHashSet();
for (String database : db2tables.keySet()) {
- List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient);
+ List<String> loaded = extractHiveTables(database, db2tables.get(database), config);
loadedTables.addAll(loaded);
}
@@ -85,29 +82,19 @@ public class HiveSourceTableLoader {
metaMgr.removeTableExd(hiveTable);
}
- private static List<String> extractHiveTables(String database, Set<String> tables, HiveClient hiveClient) throws IOException {
+ private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException {
List<String> loadedTables = Lists.newArrayList();
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
for (String tableName : tables) {
- Table table = null;
- List<FieldSchema> partitionFields = null;
- List<FieldSchema> fields = null;
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ HiveTableMeta hiveTableMeta;
try {
- table = hiveClient.getHiveTable(database, tableName);
- partitionFields = table.getPartitionKeys();
- fields = hiveClient.getHiveTableFields(database, tableName);
+ hiveTableMeta = hiveClient.getHiveTableMeta(database, tableName);
} catch (Exception e) {
- e.printStackTrace();
- throw new IOException(e);
+ throw new RuntimeException("cannot get HiveTableMeta", e);
}
- if (fields != null && partitionFields != null && partitionFields.size() > 0) {
- fields.addAll(partitionFields);
- }
-
- long tableSize = hiveClient.getFileSizeForTable(table);
- long tableFileNum = hiveClient.getFileNumberForTable(table);
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);
if (tableDesc == null) {
tableDesc = new TableDesc();
@@ -116,33 +103,32 @@ public class HiveSourceTableLoader {
tableDesc.setUuid(UUID.randomUUID().toString());
tableDesc.setLastModified(0);
}
- if (table.getTableType() != null) {
- tableDesc.setTableType(table.getTableType());
+ if (hiveTableMeta.tableType != null) {
+ tableDesc.setTableType(hiveTableMeta.tableType);
}
- int columnNumber = fields.size();
+ int columnNumber = hiveTableMeta.allColumns.size();
List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);
for (int i = 0; i < columnNumber; i++) {
- FieldSchema field = fields.get(i);
+ HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(field.getName().toUpperCase());
+ cdesc.setName(field.name.toUpperCase());
// use "double" in kylin for "float"
- if ("float".equalsIgnoreCase(field.getType())) {
+ if ("float".equalsIgnoreCase(field.dataType)) {
cdesc.setDatatype("double");
} else {
- cdesc.setDatatype(field.getType());
+ cdesc.setDatatype(field.dataType);
}
cdesc.setId(String.valueOf(i + 1));
- cdesc.setComment(field.getComment());
columns.add(cdesc);
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
StringBuffer partitionColumnString = new StringBuffer();
- for (int i = 0, n = partitionFields.size(); i < n; i++) {
+ for (int i = 0, n = hiveTableMeta.partitionColumns.size(); i < n; i++) {
if (i > 0)
partitionColumnString.append(", ");
- partitionColumnString.append(partitionFields.get(i).getName().toUpperCase());
+ partitionColumnString.append(hiveTableMeta.partitionColumns.get(i).name.toUpperCase());
}
Map<String, String> map = metaMgr.getTableDescExd(tableDesc.getIdentity());
@@ -150,16 +136,16 @@ public class HiveSourceTableLoader {
if (map == null) {
map = Maps.newHashMap();
}
- map.put(MetadataConstants.TABLE_EXD_TABLENAME, table.getTableName());
- map.put(MetadataConstants.TABLE_EXD_LOCATION, table.getSd().getLocation());
- map.put(MetadataConstants.TABLE_EXD_IF, table.getSd().getInputFormat());
- map.put(MetadataConstants.TABLE_EXD_OF, table.getSd().getOutputFormat());
- map.put(MetadataConstants.TABLE_EXD_OWNER, table.getOwner());
- map.put(MetadataConstants.TABLE_EXD_LAT, String.valueOf(table.getLastAccessTime()));
+ map.put(MetadataConstants.TABLE_EXD_TABLENAME, hiveTableMeta.tableName);
+ map.put(MetadataConstants.TABLE_EXD_LOCATION, hiveTableMeta.sdLocation);
+ map.put(MetadataConstants.TABLE_EXD_IF, hiveTableMeta.sdInputFormat);
+ map.put(MetadataConstants.TABLE_EXD_OF, hiveTableMeta.sdOutputFormat);
+ map.put(MetadataConstants.TABLE_EXD_OWNER, hiveTableMeta.owner);
+ map.put(MetadataConstants.TABLE_EXD_LAT, String.valueOf(hiveTableMeta.lastAccessTime));
map.put(MetadataConstants.TABLE_EXD_PC, partitionColumnString.toString());
- map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(tableSize));
- map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(tableFileNum));
- map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionFields != null && partitionFields.size() > 0).toString());
+ map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(hiveTableMeta.fileSize));
+ map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(hiveTableMeta.fileNum));
+ map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(hiveTableMeta.partitionColumns.size() > 0).toString());
metaMgr.saveSourceTable(tableDesc);
metaMgr.saveTableExd(tableDesc.getIdentity(), map);
@@ -168,4 +154,5 @@ public class HiveSourceTableLoader {
return loadedTables;
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
index dcc43ff..97e9990 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
@@ -37,11 +37,17 @@ public class HiveTable implements ReadableTable {
final private String database;
final private String hiveTable;
- private HiveClient hiveClient;
+ private IHiveClient hiveClient;
+ private HiveTableMeta hiveTableMeta;
public HiveTable(TableDesc tableDesc) {
this.database = tableDesc.getDatabase();
this.hiveTable = tableDesc.getName();
+ try {
+ this.hiveTableMeta = getHiveClient().getHiveTableMeta(database, hiveTable);
+ } catch (Exception e) {
+ throw new RuntimeException("cannot get HiveTableMeta", e);
+ }
}
@Override
@@ -58,7 +64,7 @@ public class HiveTable implements ReadableTable {
long lastModified = sizeAndLastModified.getSecond();
// for non-native hive table, cannot rely on size & last modified on HDFS
- if (getHiveClient().isNativeTable(database, hiveTable) == false) {
+ if (this.hiveTableMeta.isNative == false) {
lastModified = System.currentTimeMillis(); // assume table is ever changing
}
@@ -80,13 +86,13 @@ public class HiveTable implements ReadableTable {
return override;
}
- return getHiveClient().getHiveTableLocation(database, hiveTable);
+ return this.hiveTableMeta.sdLocation;
}
- public HiveClient getHiveClient() {
+ public IHiveClient getHiveClient() {
if (hiveClient == null) {
- hiveClient = new HiveClient();
+ hiveClient = HiveClientFactory.getHiveClient();
}
return hiveClient;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
new file mode 100644
index 0000000..c2b7c96
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.util.List;
+
+class HiveTableMeta {
+ static class HiveTableColumnMeta {
+ String name;
+ String dataType;
+
+ public HiveTableColumnMeta(String name, String dataType) {
+ this.name = name;
+ this.dataType = dataType;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + '}';
+ }
+ }
+
+ String tableName;
+ String sdLocation;//sd is short for storage descriptor
+ String sdInputFormat;
+ String sdOutputFormat;
+ String owner;
+ String tableType;
+ int lastAccessTime;
+ long fileSize;
+ long fileNum;
+ boolean isNative;
+ List<HiveTableColumnMeta> allColumns;
+ List<HiveTableColumnMeta> partitionColumns;
+
+ public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
+ this.tableName = tableName;
+ this.sdLocation = sdLocation;
+ this.sdInputFormat = sdInputFormat;
+ this.sdOutputFormat = sdOutputFormat;
+ this.owner = owner;
+ this.tableType = tableType;
+ this.lastAccessTime = lastAccessTime;
+ this.fileSize = fileSize;
+ this.fileNum = fileNum;
+ this.isNative = isNative;
+ this.allColumns = allColumns;
+ this.partitionColumns = partitionColumns;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveTableMeta{" + "tableName='" + tableName + '\'' + ", sdLocation='" + sdLocation + '\'' + ", sdInputFormat='" + sdInputFormat + '\'' + ", sdOutputFormat='" + sdOutputFormat + '\'' + ", owner='" + owner + '\'' + ", tableType='" + tableType + '\'' + ", lastAccessTime=" + lastAccessTime + ", fileSize=" + fileSize + ", fileNum=" + fileNum + ", isNative=" + isNative + ", allColumns=" + allColumns + ", partitionColumns=" + partitionColumns + '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
new file mode 100644
index 0000000..7a3e5d6
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+public class HiveTableMetaBuilder {
+ private String tableName;
+ private String sdLocation;
+ private String sdInputFormat;
+ private String sdOutputFormat;
+ private String owner;
+ private String tableType;
+ private int lastAccessTime;
+ private long fileSize;
+ private long fileNum;
+ private boolean isNative = true;
+ private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+
+ public HiveTableMetaBuilder setTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdLocation(String sdLocation) {
+ this.sdLocation = sdLocation;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdInputFormat(String sdInputFormat) {
+ this.sdInputFormat = sdInputFormat;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdOutputFormat(String sdOutputFormat) {
+ this.sdOutputFormat = sdOutputFormat;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setOwner(String owner) {
+ this.owner = owner;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setTableType(String tableType) {
+ this.tableType = tableType;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setLastAccessTime(int lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setFileNum(long fileNum) {
+ this.fileNum = fileNum;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setIsNative(boolean isNative) {
+ this.isNative = isNative;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setAllColumns(List<HiveTableMeta.HiveTableColumnMeta> allColumns) {
+ this.allColumns = allColumns;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setPartitionColumns(List<HiveTableMeta.HiveTableColumnMeta> partitionColumns) {
+ this.partitionColumns = partitionColumns;
+ return this;
+ }
+
+ public HiveTableMeta createHiveTableMeta() {
+ return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
deleted file mode 100644
index 79493a4..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.datanucleus.store.types.backed.HashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class HqlExecutable extends AbstractExecutable {
-
- private static final Logger logger = LoggerFactory.getLogger(HqlExecutable.class);
-
- private static final String HQL = "hql";
- private static final String HIVE_CONFIG = "hive-config";
-
- public HqlExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Map<String, String> configMap = getConfiguration();
- HiveClient hiveClient = new HiveClient(configMap);
-
- for (String hql : getHqls()) {
- hiveClient.executeHQL(hql);
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED);
- } catch (Exception e) {
- logger.error("error run hive query:" + getHqls(), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setConfiguration(Map<String, String> configMap) {
- if (configMap != null) {
- String configStr = "";
- try {
- configStr = JsonUtil.writeValueAsString(configMap);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- setParam(HIVE_CONFIG, configStr);
- }
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, String> getConfiguration() {
- String configStr = getParam(HIVE_CONFIG);
- Map<String, String> result = null;
- if (configStr != null) {
- try {
- result = JsonUtil.readValue(configStr, HashMap.class);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- return result;
- }
-
- public void setHqls(List<String> hqls) {
- setParam(HQL, StringUtils.join(hqls, ";"));
- }
-
- private List<String> getHqls() {
- final String hqls = getParam(HQL);
- if (hqls != null) {
- return Lists.newArrayList(StringUtils.split(hqls, ";"));
- } else {
- return Collections.emptyList();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
new file mode 100644
index 0000000..f218cce
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface IHiveClient {
+ void executeHQL(String hql) throws CommandNeedRetryException, IOException;
+
+ void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException;
+
+ HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception;
+
+ List<String> getHiveDbNames() throws Exception;
+
+ List<String> getHiveTableNames(String database) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
new file mode 100644
index 0000000..84da0a2
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class BeelineOptionsProcessorTest {
+ @Ignore
+ @Test
+ public void foo() {
+ String param = "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000'";
+ BeelineOptionsProcessor processor = new BeelineOptionsProcessor();
+ CommandLine commandLine = processor.process(StringUtils.split(param));
+ String n = commandLine.getOptionValue('n');
+ String u = commandLine.getOptionValue('u');
+ String p = commandLine.getOptionValue('p');
+
+ }
+}