You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/10/30 10:43:42 UTC
[01/26] kylin git commit: KYLIN-1971 TblColRef != ColumnDesc,
turns on [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1971 ab23cc242 -> daf359d38 (forced update)
KYLIN-1971 TblColRef != ColumnDesc, turns on
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea3d02ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea3d02ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea3d02ff
Branch: refs/heads/KYLIN-1971
Commit: ea3d02ff16fe96b0865b81d60063adbab3e462d2
Parents: 69213e9
Author: Li Yang <li...@apache.org>
Authored: Wed Oct 26 18:24:15 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Thu Oct 27 08:15:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 6 +
.../org/apache/kylin/cube/model/CubeDesc.java | 14 ++-
.../apache/kylin/metadata/model/JoinDesc.java | 2 +-
.../apache/kylin/metadata/model/TblColRef.java | 4 +-
.../metadata/realization/IRealization.java | 3 +
.../kylin/storage/hybrid/HybridInstance.java | 17 +++
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../relnode/OLAPToEnumerableConverter.java | 53 +++++----
.../kylin/query/routing/ModelChooser.java | 111 ++++++++++++-------
.../common/coprocessor/CoprocessorRowType.java | 3 +
.../v1/coprocessor/observer/RowTypeTest.java | 5 +-
11 files changed, 146 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 2ccdde7..6bd3826 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.LookupDesc;
@@ -395,6 +396,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ return getDescriptor().listAllColumnDescs();
+ }
+
+ @Override
public long getDateRangeStart() {
List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5c73f21..0c0b4f3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -54,6 +54,7 @@ import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
@@ -159,8 +160,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
@JsonProperty("override_kylin_properties")
private LinkedHashMap<String, String> overrideKylinProps = new LinkedHashMap<String, String>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
+ private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
+ private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
+ private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
@@ -191,6 +193,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public Set<TblColRef> listAllColumns() {
return allColumns;
}
+
+ public Set<ColumnDesc> listAllColumnDescs() {
+ return allColumnDescs;
+ }
/**
* @return dimension columns including derived, BUT NOT measures
@@ -549,6 +555,10 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size());
initDictionaryDesc();
+
+ for (TblColRef col : allColumns) {
+ allColumnDescs.add(col.getColumnDesc());
+ }
}
public void validateAggregationGroups() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index 04fbf62..5beda0a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -154,7 +154,7 @@ public class JoinDesc {
int match = 0;
for (int i = 0; i < a.length; i++) {
for (int j = 0; j < b.length; j++) {
- if (a[i].equals(b[j])) {
+ if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) {
match++;
break;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 8841ee5..f44972c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -183,8 +183,8 @@ public class TblColRef implements Serializable {
return false;
if (!StringUtils.equals(column.getName(), other.column.getName()))
return false;
-// if ((table == null ? other.table == null : table.equals(other.table)) == false)
-// return false;
+ if ((table == null ? other.table == null : table.equals(other.table)) == false)
+ return false;
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index 343ec99..f1770d7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -43,6 +44,8 @@ public interface IRealization extends IStorageAware {
public String getFactTable();
public Set<TblColRef> getAllColumns();
+
+ public Set<ColumnDesc> getAllColumnDescs();
public List<TblColRef> getAllDimensions();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 57d14d5..d7007ac 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -68,6 +69,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
private volatile IRealization[] realizations = null;
private List<TblColRef> allDimensions = null;
private Set<TblColRef> allColumns = null;
+ private Set<ColumnDesc> allColumnDescs = null;
private List<MeasureDesc> allMeasures = null;
private long dateRangeStart;
private long dateRangeEnd;
@@ -138,6 +140,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
allDimensions = Lists.newArrayList(dimensions);
allColumns = columns;
+ allColumnDescs = asColumnDescs(allColumns);
allMeasures = Lists.newArrayList(measures);
Collections.sort(realizationList, new Comparator<IRealization>() {
@@ -166,6 +169,14 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
}
+ private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) {
+ LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>();
+ for (TblColRef col : columns) {
+ result.add(col.getColumnDesc());
+ }
+ return result;
+ }
+
@Override
public CapabilityResult isCapable(SQLDigest digest) {
CapabilityResult result = new CapabilityResult();
@@ -210,6 +221,12 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
}
@Override
+ public Set<ColumnDesc> getAllColumnDescs() {
+ init();
+ return allColumnDescs;
+ }
+
+ @Override
public List<MeasureDesc> getMeasures() {
init();
return allMeasures;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 2ec5324..2065bff 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -154,7 +154,7 @@ public class ITKylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql/query01.sql";
+ String queryFileName = getQueryFolderPrefix() + "src/test/resources/query/sql_subquery/query11.sql";
File sqlFile = new File(queryFileName);
if (sqlFile.exists()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 24fc430..31cfc36 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -48,10 +48,10 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.routing.ModelChooser;
-import org.apache.kylin.query.routing.NoRealizationFoundException;
import org.apache.kylin.query.routing.QueryRouter;
import org.apache.kylin.query.schema.OLAPTable;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
/**
@@ -79,30 +79,23 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
OLAPRel.OLAPImplementor olapImplementor = new OLAPRel.OLAPImplementor();
olapImplementor.visitChild(getInput(), this);
- // find cube from olap context and apply cell level security
- try {
- for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
- // Context has no table scan is created by OLAPJoinRel which looks like
- // (sub-query) as A join (sub-query) as B
- // No realization needed for such context.
- if (context.firstTableScan == null) {
- continue;
- }
-
- Set<IRealization> candidates = ModelChooser.selectModel(context);
- IRealization realization = QueryRouter.selectRealization(context, candidates);
- context.realization = realization;
-
- doAccessControl(context);
- }
- } catch (NoRealizationFoundException e) {
- OLAPContext ctx0 = (OLAPContext) OLAPContext.getThreadLocalContexts().toArray()[0];
- if (ctx0 != null && ctx0.olapSchema.hasStarSchemaUrl()) {
- // generate hive result
- return buildHiveResult(enumImplementor, pref, ctx0);
- } else {
- throw e;
+ // identify model
+ List<OLAPContext> contexts = listContextsHavingScan();
+ Set<IRealization> candidates = ModelChooser.selectModel(contexts);
+
+ // identify realization for each context
+ for (OLAPContext context : OLAPContext.getThreadLocalContexts()) {
+
+ // Context has no table scan is created by OLAPJoinRel which looks like
+ // (sub-query) as A join (sub-query) as B
+ // No realization needed for such context.
+ if (context.firstTableScan == null) {
+ continue;
}
+
+ IRealization realization = QueryRouter.selectRealization(context, candidates);
+ context.realization = realization;
+ doAccessControl(context);
}
// rewrite query if necessary
@@ -123,6 +116,17 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
return impl.visitChild(this, 0, inputAsEnum, pref);
}
+ private List<OLAPContext> listContextsHavingScan() {
+ int size = OLAPContext.getThreadLocalContexts().size();
+ List<OLAPContext> result = Lists.newArrayListWithCapacity(size);
+ for (int i = 0; i < size; i++) {
+ OLAPContext ctx = OLAPContext.getThreadLocalContextById(i);
+ if (ctx.firstTableScan != null)
+ result.add(ctx);
+ }
+ return result;
+ }
+
private void doAccessControl(OLAPContext context) {
String controllerCls = KylinConfig.getInstanceFromEnv().getQueryAccessController();
if (null != controllerCls && !controllerCls.isEmpty()) {
@@ -176,6 +180,7 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
}
}
+ @SuppressWarnings("unused")
private Result buildHiveResult(EnumerableRelImplementor enumImplementor, Prefer pref, OLAPContext context) {
RelDataType hiveRowType = getRowType();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index 4667f4a..08616bd 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -26,10 +26,12 @@ import java.util.TreeMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.LookupDesc;
import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -42,55 +44,68 @@ import com.google.common.collect.Sets;
public class ModelChooser {
- public static Set<IRealization> selectModel(OLAPContext context) {
- Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(context);
- OLAPTableScan firstTable = context.firstTableScan;
- List<JoinDesc> joins = context.joins;
+ // select a model that satisfies all the contexts
+ public static Set<IRealization> selectModel(List<OLAPContext> contexts) {
+ Map<DataModelDesc, Set<IRealization>> modelMap = makeOrderedModelMap(contexts);
for (DataModelDesc model : modelMap.keySet()) {
- Map<String, String> aliasMap = matches(model, firstTable, joins);
+ Map<String, String> aliasMap = matches(model, contexts);
if (aliasMap != null) {
- fixModel(context, model, aliasMap);
+ for (OLAPContext ctx : contexts)
+ fixModel(ctx, model, aliasMap);
return modelMap.get(model);
}
}
-
- throw new NoRealizationFoundException("No model found by first table " + firstTable.getOlapTable().getTableName() + " and joins " + joins);
+
+ throw new NoRealizationFoundException("No model found for" + toErrorMsg(contexts));
}
- private static Map<String, String> matches(DataModelDesc model, OLAPTableScan firstTable, List<JoinDesc> joins) {
- Map<String, String> result = Maps.newHashMap();
-
- // no join special case
- if (joins.isEmpty()) {
- TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
- if (tableRef == null)
- return null;
- result.put(firstTable.getAlias(), tableRef.getAlias());
- return result;
+ private static String toErrorMsg(List<OLAPContext> contexts) {
+ StringBuilder buf = new StringBuilder();
+ for (OLAPContext ctx : contexts) {
+ buf.append(", ").append(ctx.firstTableScan);
+ for (JoinDesc join : ctx.joins)
+ buf.append(", ").append(join);
}
-
+ return buf.toString();
+ }
+
+ private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) {
+ Map<String, String> result = Maps.newHashMap();
+
// the greedy match is not perfect but works for the moment
Map<String, List<JoinDesc>> modelJoinsMap = model.getJoinsMap();
- for (JoinDesc queryJoin : joins) {
- String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
- List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
- if (modelJoins == null)
- return null;
-
- JoinDesc matchJoin = null;
- for (JoinDesc modelJoin : modelJoins) {
- if (modelJoin.matches(queryJoin)) {
- matchJoin = modelJoin;
- break;
+ for (OLAPContext ctx : contexts) {
+ for (JoinDesc queryJoin : ctx.joins) {
+ String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
+ List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
+ if (modelJoins == null)
+ return null;
+
+ JoinDesc matchJoin = null;
+ for (JoinDesc modelJoin : modelJoins) {
+ if (modelJoin.matches(queryJoin)) {
+ matchJoin = modelJoin;
+ break;
+ }
}
+ if (matchJoin == null)
+ return null;
+
+ matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
+ matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
}
- if (matchJoin == null)
- return null;
- matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
- matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
+ OLAPTableScan firstTable = ctx.firstTableScan;
+ String firstTableAlias = firstTable.getAlias();
+ if (result.containsKey(firstTableAlias) == false) {
+ TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
+ if (tableRef == null)
+ return null;
+ matchesAdd(firstTableAlias, tableRef.getAlias(), result);
+ }
}
+
return result;
}
@@ -99,10 +114,12 @@ public class ModelChooser {
Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
}
- private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(OLAPContext context) {
- KylinConfig kylinConfig = context.olapSchema.getConfig();
- String projectName = context.olapSchema.getProjectName();
- String factTableName = context.firstTableScan.getOlapTable().getTableName();
+ private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) {
+ // the first context, which is the top most context, contains all columns from all contexts
+ OLAPContext first = contexts.get(0);
+ KylinConfig kylinConfig = first.olapSchema.getConfig();
+ String projectName = first.olapSchema.getProjectName();
+ String factTableName = first.firstTableScan.getOlapTable().getTableName();
Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName);
final Map<DataModelDesc, Set<IRealization>> models = Maps.newHashMap();
@@ -110,7 +127,7 @@ public class ModelChooser {
for (IRealization real : realizations) {
if (real.isReady() == false)
continue;
- if (real.getAllColumns().containsAll(context.allColumns) == false)
+ if (containsAll(real.getAllColumnDescs(), first.allColumns) == false)
continue;
if (RemoveBlackoutRealizationsRule.accept(real) == false)
continue;
@@ -143,20 +160,28 @@ public class ModelChooser {
return result;
}
+ private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
+ for (TblColRef col : allColumns) {
+ if (allColumnDescs.contains(col.getColumnDesc()) == false)
+ return false;
+ }
+ return true;
+ }
+
private static void fixModel(OLAPContext context, DataModelDesc model, Map<String, String> aliasMap) {
for (OLAPTableScan tableScan : context.allTableScans) {
tableScan.fixColumnRowTypeWithModel(model, aliasMap);
}
}
-
+
private static class RealizationCost implements Comparable<RealizationCost> {
final public int priority;
final public int cost;
-
+
public RealizationCost(IRealization real) {
// ref Candidate.PRIORITIES
this.priority = Candidate.PRIORITIES.get(real.getType());
-
+
// ref CubeInstance.getCost()
int c = real.getAllDimensions().size() * CubeInstance.COST_WEIGHT_DIMENSION + real.getMeasures().size() * CubeInstance.COST_WEIGHT_MEASURE;
for (LookupDesc lookup : real.getDataModelDesc().getLookups()) {
@@ -165,7 +190,7 @@ public class ModelChooser {
}
this.cost = c;
}
-
+
@Override
public int compareTo(RealizationCost o) {
int comp = this.priority - o.priority;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
index f1e26f5..7314fc4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java
@@ -73,6 +73,7 @@ public class CoprocessorRowType {
for (int i = 0; i < n; i++) {
BytesUtil.writeAsciiString(o.columns[i].getTable(), out);
BytesUtil.writeAsciiString(o.columns[i].getName(), out);
+ BytesUtil.writeAsciiString(o.columns[i].getDatatype(), out);
BytesUtil.writeVInt(o.columnSizes[i], out);
}
}
@@ -86,11 +87,13 @@ public class CoprocessorRowType {
for (int i = 0; i < n; i++) {
String tableName = BytesUtil.readAsciiString(in);
String colName = BytesUtil.readAsciiString(in);
+ String datatype = BytesUtil.readAsciiString(in);
TableDesc table = new TableDesc();
table.setName(tableName);
ColumnDesc col = new ColumnDesc();
col.setTable(table);
col.setName(colName);
+ col.setDatatype(datatype);
col.init(table);
cols[i] = col.getRef();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea3d02ff/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
index ba4b86a..364be9a 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
@@ -60,7 +61,9 @@ public class RowTypeTest extends LocalFileMetadataTestCase {
byte[] bytes = CoprocessorRowType.serialize(rowType);
CoprocessorRowType copy = CoprocessorRowType.deserialize(bytes);
- assertTrue(Arrays.equals(rowType.columns, copy.columns));
assertTrue(Arrays.equals(rowType.columnSizes, copy.columnSizes));
+ for (int i = 0; i < rowType.columns.length; i++) {
+ assertEquals(rowType.columns[i].getColumnDesc(), copy.columns[i].getColumnDesc());
+ }
}
}
[21/26] kylin git commit: KYLIN-2136 Enhance cubing algorithm
selection
Posted by li...@apache.org.
KYLIN-2136 Enhance cubing algorithm selection
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0cfaa1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0cfaa1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0cfaa1b
Branch: refs/heads/KYLIN-1971
Commit: a0cfaa1bcba920b98b663f19e4df8142019d5965
Parents: 5922830
Author: Li Yang <li...@apache.org>
Authored: Fri Oct 28 11:34:14 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Oct 28 14:52:05 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 13 +++-
.../kylin/engine/mr/common/CubeStatsWriter.java | 76 ++++++++++++++++++++
.../kylin/engine/mr/common/CuboidStatsUtil.java | 73 -------------------
.../mr/steps/FactDistinctColumnsReducer.java | 9 ++-
.../engine/mr/steps/MergeStatisticsStep.java | 4 +-
.../engine/mr/steps/SaveStatisticsStep.java | 17 +++--
.../steps/FactDistinctColumnsReducerTest.java | 4 +-
8 files changed, 115 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8744296..47f9878 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -424,9 +424,13 @@ abstract public class KylinConfigBase implements Serializable {
}
public double getCubeAlgorithmAutoThreshold() {
- return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
+ return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "7"));
}
+ public int getCubeAlgorithmAutoMapperLimit() {
+ return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500"));
+ }
+
@Deprecated
public int getCubeAggrGroupMaxSize() {
return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 83e46e3..2dcb268 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -73,7 +73,8 @@ public class CubeStatsReader {
final CubeSegment seg;
final int samplingPercentage;
- final double mapperOverlapRatioOfFirstBuild; // only makes sense for the first build, is meaningless after merge
+ final int mapperNumberOfFirstBuild; // becomes meaningless after merge
+ final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after merge
final Map<Long, HyperLogLogPlusCounter> cuboidRowEstimatesHLL;
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
@@ -90,6 +91,7 @@ public class CubeStatsReader {
reader = new SequenceFile.Reader(hadoopConf, seqInput);
int percentage = 100;
+ int mapperNumber = 0;
double mapperOverlapRatio = 0;
Map<Long, HyperLogLogPlusCounter> counterMap = Maps.newHashMap();
@@ -100,7 +102,9 @@ public class CubeStatsReader {
percentage = Bytes.toInt(value.getBytes());
} else if (key.get() == -1) {
mapperOverlapRatio = Bytes.toDouble(value.getBytes());
- } else {
+ } else if (key.get() == -2) {
+ mapperNumber = Bytes.toInt(value.getBytes());
+ } else if (key.get() > 0) {
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(kylinConfig.getCubeStatsHLLPrecision());
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
@@ -110,6 +114,7 @@ public class CubeStatsReader {
this.seg = cubeSegment;
this.samplingPercentage = percentage;
+ this.mapperNumberOfFirstBuild = mapperNumber;
this.mapperOverlapRatioOfFirstBuild = mapperOverlapRatio;
this.cuboidRowEstimatesHLL = counterMap;
@@ -141,6 +146,10 @@ public class CubeStatsReader {
return getCuboidSizeMapFromRowCount(seg, getCuboidRowEstimatesHLL());
}
+ public int getMapperNumberOfFirstBuild() {
+ return mapperNumberOfFirstBuild;
+ }
+
public double getMapperOverlapRatioOfFirstBuild() {
return mapperOverlapRatioOfFirstBuild;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
new file mode 100644
index 0000000..74a2107
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+
+public class CubeStatsWriter {
+
+ public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
+ Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
+ writeCuboidStatistics(conf, outputPath, cuboidHLLMap, samplingPercentage, 0, 0);
+ }
+
+ public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
+ Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio) throws IOException {
+ Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
+
+ List<Long> allCuboids = new ArrayList<Long>();
+ allCuboids.addAll(cuboidHLLMap.keySet());
+ Collections.sort(allCuboids);
+
+ ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
+ SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class), SequenceFile.Writer.valueClass(BytesWritable.class));
+ try {
+ // mapper overlap ratio at key -1
+ writer.append(new LongWritable(-1), new BytesWritable(Bytes.toBytes(mapperOverlapRatio)));
+
+ // mapper number at key -2
+ writer.append(new LongWritable(-2), new BytesWritable(Bytes.toBytes(mapperNumber)));
+
+ // sampling percentage at key 0
+ writer.append(new LongWritable(0L), new BytesWritable(Bytes.toBytes(samplingPercentage)));
+
+ for (long i : allCuboids) {
+ valueBuf.clear();
+ cuboidHLLMap.get(i).writeRegisters(valueBuf);
+ valueBuf.flip();
+ writer.append(new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()));
+ }
+ } finally {
+ IOUtils.closeQuietly(writer);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
deleted file mode 100644
index d5b4b0d..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.engine.mr.common;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-
-public class CuboidStatsUtil {
-
- public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
- writeCuboidStatistics(conf, outputPath, cuboidHLLMap, samplingPercentage, 0);
- }
-
- public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage, double mapperOverlapRatio) throws IOException {
- Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
-
- List<Long> allCuboids = new ArrayList<Long>();
- allCuboids.addAll(cuboidHLLMap.keySet());
- Collections.sort(allCuboids);
-
- ByteBuffer valueBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
- SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class), SequenceFile.Writer.valueClass(BytesWritable.class));
- try {
- // mapper overlap ratio at key -1
- writer.append(new LongWritable(-1), new BytesWritable(Bytes.toBytes(mapperOverlapRatio)));
-
- // sampling percentage at key 0
- writer.append(new LongWritable(0L), new BytesWritable(Bytes.toBytes(samplingPercentage)));
-
- for (long i : allCuboids) {
- valueBuf.clear();
- cuboidHLLMap.get(i).writeRegisters(valueBuf);
- valueBuf.flip();
- writer.append(new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()));
- }
- } finally {
- IOUtils.closeQuietly(writer);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 2889ba8..c8624bb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -41,7 +41,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -189,10 +189,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
grandTotal += hll.getCountEstimate();
}
double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
+
+ int mapperNumber = baseCuboidRowCountInMappers.size();
writeMapperAndCuboidStatistics(context); // for human check
- CuboidStatsUtil.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
- cuboidHLLMap, samplingPercentage, mapperOverlapRatio); // for CreateHTableJob
+ CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
+ cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
}
}
@@ -214,6 +216,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
writeLine(out, msg);
writeLine(out, "The following statistics are collected based on sampling data.");
+ writeLine(out, "Number of Mappers: " + baseCuboidRowCountInMappers.size());
for (int i = 0; i < baseCuboidRowCountInMappers.size(); i++) {
if (baseCuboidRowCountInMappers.get(i) > 0) {
msg = "Base Cuboid in Mapper " + i + " row count: \t " + baseCuboidRowCountInMappers.get(i);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index c774cd6..88f6ba2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -42,7 +42,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -121,7 +121,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
}
}
averageSamplingPercentage = averageSamplingPercentage / CubingExecutableUtil.getMergingSegmentIds(this.getParams()).size();
- CuboidStatsUtil.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage);
+ CubeStatsWriter.writeCuboidStatistics(conf, new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams())), cuboidHLLMap, averageSamplingPercentage);
Path statisticsFilePath = new Path(CubingExecutableUtil.getMergedStatisticsPath(this.getParams()), BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
FileSystem fs = statisticsFilePath.getFileSystem(conf);
FSDataInputStream is = fs.open(statisticsFilePath);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 3cace64..8777af7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -104,10 +104,19 @@ public class SaveStatisticsStep extends AbstractExecutable {
} else if ("random".equalsIgnoreCase(algPref)) { // for testing
alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
} else { // the default
- double threshold = kylinConf.getCubeAlgorithmAutoThreshold();
- double mapperOverlapRatio = new CubeStatsReader(seg, kylinConf).getMapperOverlapRatioOfFirstBuild();
- logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + threshold);
- alg = mapperOverlapRatio < threshold ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
+ CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
+ int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
+ int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
+ double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
+ double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
+ logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
+ logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
+
+ // in-mem cubing is good when
+ // 1) the cluster has enough mapper slots to run in parallel
+ // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
+ alg = (mapperNumber <= mapperNumLimit && mapperOverlapRatio <= overlapThreshold)//
+ ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0cfaa1b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index cbbaf38..ca8684f 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.engine.mr.common.CuboidStatsUtil;
+import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.junit.Test;
@@ -49,7 +49,7 @@ public class FactDistinctColumnsReducerTest {
System.out.println(outputPath);
Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
- CuboidStatsUtil.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
+ CubeStatsWriter.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
FileSystem.getLocal(conf).delete(outputPath, true);
}
[25/26] kylin git commit: minor refactor of CubeSegment
Posted by li...@apache.org.
minor refactor of CubeSegment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/635b3b97
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/635b3b97
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/635b3b97
Branch: refs/heads/KYLIN-1971
Commit: 635b3b97662515b7b9e4c58d6996c28c12fcfb28
Parents: 650f57c
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 30 15:45:17 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 30 15:45:17 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeSegment.java | 33 +++++++-------------
1 file changed, 11 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/635b3b97/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index b3d6d6b..1ec01a2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -42,15 +42,13 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import javax.annotation.concurrent.GuardedBy;
-
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
@@ -117,8 +115,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private Map<String, String> additionalInfo = new LinkedHashMap<String, String>();
- @GuardedBy("this")
- private Map<Long, Short> cuboidBaseShards = Maps.newHashMap(); // cuboid id ==> base(starting) shard for this cuboid
+ private Map<Long, Short> cuboidBaseShards = Maps.newConcurrentMap(); // cuboid id ==> base(starting) shard for this cuboid
public CubeDesc getCubeDesc() {
return getCubeInstance().getDescriptor();
@@ -505,12 +502,9 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
public int getTotalShards(long cuboidId) {
if (totalShards > 0) {
- //shard squashed case
- //logger.info("total shards for {} is {}", cuboidId, totalShards);
return totalShards;
} else {
int ret = getCuboidShardNum(cuboidId);
- //logger.info("total shards for {} is {}", cuboidId, ret);
return ret;
}
}
@@ -519,22 +513,17 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
this.totalShards = totalShards;
}
- public synchronized short getCuboidBaseShard(Long cuboidId) {
- if (totalShards > 0) {
- //shard squashed case
-
- Short ret = cuboidBaseShards.get(cuboidId);
- if (ret == null) {
- ret = ShardingHash.getShard(cuboidId, totalShards);
- cuboidBaseShards.put(cuboidId, ret);
- }
-
- //logger.info("base for cuboid {} is {}", cuboidId, ret);
- return ret;
- } else {
- //logger.info("base for cuboid {} is {}", cuboidId, 0);
+ public short getCuboidBaseShard(Long cuboidId) {
+ if (totalShards == 0)
return 0;
+
+ Short ret = cuboidBaseShards.get(cuboidId);
+ if (ret == null) {
+ ret = ShardingHash.getShard(cuboidId, totalShards);
+ cuboidBaseShards.put(cuboidId, ret);
}
+
+ return ret;
}
public List<Long> getBlackoutCuboids() {
[15/26] kylin git commit: minor,
remove kylin.dictionary.max.cardinality
Posted by li...@apache.org.
minor, remove kylin.dictionary.max.cardinality
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a43729bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a43729bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a43729bc
Branch: refs/heads/KYLIN-1971
Commit: a43729bc75b431bbabee49e89a4e60c60cbf6baa
Parents: 5c1c24a
Author: Li Yang <li...@apache.org>
Authored: Thu Oct 27 14:34:40 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Oct 27 14:34:40 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 4 ----
.../org/apache/kylin/dict/DictionaryGenerator.java | 14 --------------
2 files changed, 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f49127b..455b691 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -452,10 +452,6 @@ abstract public class KylinConfigBase implements Serializable {
return getPropertiesByPrefix("kylin.cube.measure.customMeasureType.");
}
- public int getDictionaryMaxCardinality() {
- return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "30000000"));
- }
-
public int getTableSnapshotMaxMB() {
return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", "300"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a43729bc/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index ba848c6..0adf40e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.datatype.DataType;
@@ -40,20 +39,10 @@ import com.google.common.base.Preconditions;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class DictionaryGenerator {
- private static final int DICT_MAX_CARDINALITY = getDictionaryMaxCardinality();
-
private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
- private static int getDictionaryMaxCardinality() {
- try {
- return KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
- } catch (Throwable e) {
- return 30000000; // some test case does not have KylinConfig setup properly
- }
- }
-
public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
Preconditions.checkNotNull(dataType, "dataType cannot be null");
@@ -92,9 +81,6 @@ public class DictionaryGenerator {
logger.debug("Dictionary cardinality: " + dict.getSize());
logger.debug("Dictionary builder class: " + builder.getClass().getName());
logger.debug("Dictionary class: " + dict.getClass().getName());
- if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
- throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- cardinality: " + dict.getSize());
- }
return dict;
}
[10/26] kylin git commit: KYLIN-2070 UI changes for allowing
concurrent build/refresh/merge
Posted by li...@apache.org.
KYLIN-2070 UI changes for allowing concurrent build/refresh/merge
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/efa945e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/efa945e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/efa945e5
Branch: refs/heads/KYLIN-1971
Commit: efa945e56b1d07648c3f0507831b769d0a018e7e
Parents: 7a793e5
Author: luguosheng <55...@qq.com>
Authored: Fri Oct 21 11:40:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 27 09:10:23 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 40 ++++++++++++++++++++++++++++-----
webapp/app/js/services/cubes.js | 1 +
2 files changed, 36 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 499e93c..f735fb8 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -334,12 +334,42 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
$scope.loadDetail(cube);
// for streaming cube build tip
if(cube.streaming){
- $modal.open({
- templateUrl: 'streamingBuild.html',
- controller: streamingBuildCtrl,
- resolve: {
+
+ SweetAlert.swal({
+ title: '',
+ text: "Are you sure to start the build?",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function(isConfirm) {
+ if(isConfirm){
+ loadingRequest.show();
+ CubeService.rebuildStreamingCube(
+ {
+ cubeId: cube.name
+ },
+ {
+ sourceOffsetStart:0,
+ sourceOffsetEnd:'9223372036854775807',
+ buildType:'BUILD'
+ }, function (job) {
+ loadingRequest.hide();
+ SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 'success');
+ },function(e){
+
+ loadingRequest.hide();
+ if(e.data&& e.data.exception){
+ var message =e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ }else{
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
}
- });
+ })
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/efa945e5/webapp/app/js/services/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 953f157..de60f55 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -26,6 +26,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
cost: {method: 'PUT', params: {action: 'cost'}, isArray: false},
rebuildLookUp: {method: 'PUT', params: {propName: 'segs', action: 'refresh_lookup'}, isArray: false},
rebuildCube: {method: 'PUT', params: {action: 'rebuild'}, isArray: false},
+ rebuildStreamingCube: {method: 'PUT', params: {action: 'build2'}, isArray: false},
disable: {method: 'PUT', params: {action: 'disable'}, isArray: false},
enable: {method: 'PUT', params: {action: 'enable'}, isArray: false},
purge: {method: 'PUT', params: {action: 'purge'}, isArray: false},
[02/26] kylin git commit: KYLIN-2030 fix with KYLIN-1971
Posted by li...@apache.org.
KYLIN-2030 fix with KYLIN-1971
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c804dc8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c804dc8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c804dc8d
Branch: refs/heads/KYLIN-1971
Commit: c804dc8d77bb0402d6cd391e582a47f323edbb21
Parents: 324e423
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 14:41:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeCapabilityChecker.java | 13 +++----------
.../org/apache/kylin/metadata/model/ColumnDesc.java | 8 +-------
2 files changed, 4 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e509d98..a3c89e5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,22 +25,18 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,11 +82,8 @@ public class CubeCapabilityChecker {
} else {
//for non query-on-facttable
if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
- TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
- Set<TblColRef> dimCols = Sets.newHashSet();
- for (ColumnDesc columnDesc : tableDesc.getColumns()) {
- dimCols.add(columnDesc.getRef());
- }
+
+ Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
//1. all aggregations on lookup table can be done. For distinct count, mark them all DimensionAsMeasures
// so that the measure has a chance to be upgraded to DimCountDistinctMeasureType in org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
http://git-wip-us.apache.org/repos/asf/kylin/blob/c804dc8d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a13bd37..e0184b4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
/**
* Column Metadata from Source. All name should be uppercase.
@@ -203,11 +202,6 @@ public class ColumnDesc implements Serializable {
@Override
public String toString() {
- return "ColumnDesc{" +
- "id='" + id + '\'' +
- ", name='" + name + '\'' +
- ", datatype='" + datatype + '\'' +
- ", comment='" + comment + '\'' +
- '}';
+ return "ColumnDesc{" + "id='" + id + '\'' + ", name='" + name + '\'' + ", datatype='" + datatype + '\'' + ", comment='" + comment + '\'' + '}';
}
}
[24/26] kylin git commit: Revert "KYLIN-2137 add check for
'kylin.job.mr.lib.dir'"
Posted by li...@apache.org.
Revert "KYLIN-2137 add check for 'kylin.job.mr.lib.dir'"
This reverts commit c5af44353d5904ddb8e9107a160872164f0e5f4d.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/650f57c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/650f57c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/650f57c8
Branch: refs/heads/KYLIN-1971
Commit: 650f57c88c1cad890c5ca8b7679d7d94e8279080
Parents: c5af443
Author: Li Yang <li...@apache.org>
Authored: Sat Oct 29 06:48:22 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Oct 29 06:48:22 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/650f57c8/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a97a8c7..417d63d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,14 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- if (!StringUtils.isBlank(mrLibDir)) {
- File dirFileMRLIB = new File(mrLibDir);
- if (dirFileMRLIB.exists()) {
- StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
- } else {
- logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!");
- }
- }
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
setJobTmpJarsAndFiles(job, kylinDependency.toString());
[23/26] kylin git commit: KYLIN-2137 add check for
'kylin.job.mr.lib.dir'
Posted by li...@apache.org.
KYLIN-2137 add check for 'kylin.job.mr.lib.dir'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5af4435
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5af4435
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5af4435
Branch: refs/heads/KYLIN-1971
Commit: c5af44353d5904ddb8e9107a160872164f0e5f4d
Parents: 8314e74
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 28 22:17:04 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 28 22:17:34 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5af4435/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index 417d63d..a97a8c7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -245,7 +245,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+ if (!StringUtils.isBlank(mrLibDir)) {
+ File dirFileMRLIB = new File(mrLibDir);
+ if (dirFileMRLIB.exists()) {
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
+ } else {
+ logger.info("The directory '" + mrLibDir + "' for 'kylin.job.mr.lib.dir' does not exist!!!");
+ }
+ }
setJobTmpJarsAndFiles(job, kylinDependency.toString());
[17/26] kylin git commit: KYLIN-2134 Kylin will treat empty string as
NULL by mistake
Posted by li...@apache.org.
KYLIN-2134 Kylin will treat empty string as NULL by mistake
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/49260dd0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/49260dd0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/49260dd0
Branch: refs/heads/KYLIN-1971
Commit: 49260dd0cb1c129d40b51c359197402441b87256
Parents: 468fe9f
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Oct 27 15:22:54 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 15:22:57 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/tuple/Tuple.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/49260dd0/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index aaf9aa9..9074403 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -178,7 +178,7 @@ public class Tuple implements ITuple {
if (strValue == null)
return null;
- if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string"))
+ if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
return null;
// TODO use data type enum instead of string comparison
[26/26] kylin git commit: minor, tool ClasspathScanner
Posted by li...@apache.org.
minor, tool ClasspathScanner
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/daf359d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/daf359d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/daf359d3
Branch: refs/heads/KYLIN-1971
Commit: daf359d38e9339c9c1151e5b90a0449d05841e19
Parents: 635b3b9
Author: Yang Li <li...@apache.org>
Authored: Sun Oct 30 18:40:51 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Oct 30 18:40:51 2016 +0800
----------------------------------------------------------------------
.../kylin/common/util/ClasspathScanner.java | 335 +++++++++++++++++++
.../apache/kylin/common/util/StringUtil.java | 16 +
2 files changed, 351 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/daf359d3/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
new file mode 100644
index 0000000..6eeb53a
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClasspathScanner.java
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+
+public class ClasspathScanner {
+
+ public static void main(final String[] args) {
+ ClasspathScanner scanner = new ClasspathScanner();
+
+ System.out.println("Finding " + Arrays.toString(args) + " in:");
+ System.out.println("----------------------------------------------------------------------------");
+ for (File f : scanner.rootResources) {
+ System.out.println(" - " + f.getAbsolutePath());
+ }
+ System.out.println("----------------------------------------------------------------------------");
+
+ scanner.scan("", new ResourceVisitor() {
+ public void accept(File dir, String relativeFileName) {
+ check(dir.getAbsolutePath(), relativeFileName.replace('\\', '/'));
+ }
+
+ public void accept(ZipFile archive, ZipEntry zipEntry) {
+ check(archive.getName(), zipEntry.getName().replace('\\', '/'));
+ }
+
+ private void check(String base, String relativePath) {
+ boolean hit = false;
+ for (int i = 0; i < args.length && !hit; i++)
+ hit = match(args[0], relativePath);
+
+ if (hit) {
+ System.out.println(base + " - " + relativePath);
+ }
+ }
+ });
+ }
+
+ /**
+ * Scan classpath to find resources that has a suffix in name.
+ *
+ * This thread's context class loader is used to define the searching
+ * classpath.
+ *
+ * @param suffix
+ * like ".jsp" for example
+ * @return a string array; each element is a standard resource name relative
+ * to the root of class path, like "young/web/frame.jsp"
+ */
+ public static String[] findResources(final String suffix) {
+ ClasspathScanner scanner = new ClasspathScanner();
+
+ final ArrayList result = new ArrayList();
+
+ scanner.scan(suffix, new ResourceVisitor() {
+ public void accept(File dir, String relativeFileName) {
+ result.add(relativeFileName.replace('\\', '/'));
+ }
+
+ public void accept(ZipFile archive, ZipEntry zipEntry) {
+ result.add(zipEntry.getName().replace('\\', '/'));
+ }
+ });
+
+ return (String[]) result.toArray(new String[result.size()]);
+ }
+
+ // ============================================================================
+
+ private File[] rootResources;
+
+ public ClasspathScanner() {
+ this(Thread.currentThread().getContextClassLoader(), true);
+ }
+
+ public ClasspathScanner(ClassLoader classLoader, boolean recursive) {
+ this.rootResources = extractRoots(classLoader, recursive);
+ }
+
+ static File[] extractRoots(ClassLoader loader, boolean recursive) {
+ ArrayList<File> roots = new ArrayList();
+
+ do {
+ if (loader instanceof URLClassLoader) {
+ URL[] urls = ((URLClassLoader) loader).getURLs();
+ for (int i = 0; i < urls.length; i++) {
+ // tricky: space is "%20" in URL
+ File f = new File(urls[i].getFile().replace("%20", " "));
+
+ // some generated run script could contain empty path, i.e., foo::bar
+ // try detect and filter them out
+ if (f.isDirectory()) {
+ boolean badSuspect = false;
+ for (File known : roots)
+ if (known.isDirectory() && known.toString().startsWith(f.toString()))
+ badSuspect = true;
+ if (badSuspect)
+ continue;
+ }
+ roots.add(f);
+ }
+ }
+ loader = loader.getParent();
+ } while (loader != null && recursive);
+
+ return (File[]) roots.toArray(new File[roots.size()]);
+ }
+
+ /**
+ * @param rootResources
+ * can be directories or zip/jar archives. All contents inside
+ * will be searched.
+ */
+ public ClasspathScanner(File[] rootResources) {
+ this.rootResources = rootResources;
+ }
+
+ public static interface ResourceVisitor {
+
+ void accept(File dir, String relativeFileName);
+
+ void accept(ZipFile archive, ZipEntry zipEntry);
+ }
+
+ public void scan(String suffix, ResourceVisitor visitor) {
+ for (int i = 0; i < rootResources.length; i++) {
+ if (rootResources[i].exists()) {
+ if (rootResources[i].isDirectory()) {
+ scanDirectory(rootResources[i], suffix, visitor);
+ } else if (rootResources[i].getName().contains(".zip") || rootResources[i].getName().contains(".jar")) {
+ scanArchive(rootResources[i], suffix, visitor);
+ }
+ }
+ }
+ }
+
+ private void scanArchive(File archive, String suffix, ResourceVisitor visitor) {
+ ZipFile zip = null;
+ try {
+ zip = new ZipFile(archive, ZipFile.OPEN_READ);
+ Enumeration enu = zip.entries();
+ while (enu.hasMoreElements()) {
+ ZipEntry entry = (ZipEntry) enu.nextElement();
+ if (entry.getName().endsWith(suffix)) {
+ visitor.accept(zip, entry);
+ }
+ }
+ } catch (ZipException e) {
+ // e.printStackTrace();
+ } catch (IOException e) {
+ // e.printStackTrace();
+ } finally {
+ if (zip != null) {
+ try {
+ zip.close();
+ } catch (IOException e) {
+ // e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private void scanDirectory(File dir, String suffix, ResourceVisitor visitor) {
+ String[] files = scanFiles(dir, new String[] { "*" + suffix });
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].endsWith(suffix)) {
+ visitor.accept(dir, files[i]);
+ }
+ }
+ }
+
+ /**
+ * Search files inside a dir that match certain patterns.
+ * @param dir from where files are searched
+ * @param includes patterns that may contain '*' or '?' wild char
+ * @return an array of included files relative to the dir
+ */
+ public static String[] scanFiles(File dir, String... includes) {
+ return scanFiles(dir, includes, null);
+ }
+
+ /**
+ * Search files inside a dir that match any of the include patterns
+ * and not match the exclude patterns. All files are included if
+ * include patterns is null/empty; no files are excluded if exclude
+ * patterns is null/empty.
+ * @param dir from where files are searched
+ * @param includes patterns that may contain '*' or '?' wild char
+ * @param excludes patterns that may contain '*' or '?' wild char
+ * @return an array of included files relative to the dir
+ */
+ public static String[] scanFiles(File dir, String[] includes, String[] excludes) {
+ // remove tailing '/' in patterns
+ if (includes != null) {
+ for (int i = 0; i < includes.length; i++)
+ includes[i] = StringUtil.trimSuffix(includes[i], "/");
+ }
+ if (excludes != null) {
+ for (int i = 0; i < excludes.length; i++)
+ excludes[i] = StringUtil.trimSuffix(excludes[i], "/");
+ }
+
+ ArrayList result = new ArrayList();
+ ArrayList queue = new ArrayList();
+ queue.add("");
+
+ String dirPath, path;
+ File dirFile, f;
+ File[] files;
+ while (!queue.isEmpty()) {
+ dirPath = (String) queue.remove(queue.size() - 1);
+ dirFile = dirPath.length() == 0 ? dir : new File(dir, dirPath);
+ files = dirFile.listFiles();
+ for (int i = 0; files != null && i < files.length; i++) {
+ f = files[i];
+ path = dirPath + (dirPath.length() == 0 ? "" : "/") + f.getName();
+ if (f.isDirectory()) {
+ // cut off excluded dir early
+ if (scanFiles_isIncluded(path, null, excludes))
+ queue.add(path);
+ } else if (f.isFile()) {
+ if (scanFiles_isIncluded(path, includes, excludes))
+ result.add(path);
+ }
+ }
+ }
+ return (String[]) result.toArray(new String[result.size()]);
+ }
+
+ private static boolean scanFiles_isIncluded(String path, String[] includes, String[] excludes) {
+ // if null, means include everything
+ if (includes != null && includes.length != 0) {
+ boolean included = false;
+ for (int i = 0; !included && i < includes.length; i++) {
+ if (match(includes[i], path) || match(includes[i] + "/*", path))
+ included = true;
+ }
+ if (!included)
+ return false;
+ }
+ // if null, means exclude nothing
+ if (excludes != null && excludes.length != 0) {
+ for (int i = 0; i < excludes.length; i++) {
+ if (match(excludes[i], path))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Matches a string against a pattern. The pattern contains two special
+ * characters: '*' which means zero or more characters, '?' which means one
+ * and only one character.
+ *
+ * @param pattern
+ * the (non-null) pattern to match against
+ * @param str
+ * the (non-null) string that must be matched against the pattern
+ *
+ * @return <code>true</code> when the string matches against the pattern,
+ * <code>false</code> otherwise.
+ */
+ public static boolean match(String pattern, String str) {
+ int i = 0, j = 0, ii = 0, jj = 0, plen = pattern.length(), slen = str.length();
+ int wordStart, wordEnd, lastPossible;
+ while (i < plen) {
+ // find the next word in pattern
+ wordStart = i;
+ while (wordStart < plen && pattern.charAt(wordStart) == '*')
+ wordStart++;
+ if (wordStart == plen) // all left in pattern is '*'
+ return true;
+ // find the word end
+ wordEnd = wordStart + 1;
+ while (wordEnd < plen && pattern.charAt(wordEnd) != '*')
+ wordEnd++;
+
+ // locate the word in string
+ lastPossible = slen - (wordEnd - wordStart) + 1;
+ for (; j < lastPossible; j++) {
+ // check if word matches at j
+ for (ii = wordStart, jj = j; ii < wordEnd; ii++, jj++)
+ if (pattern.charAt(ii) != '?' && pattern.charAt(ii) != str.charAt(jj))
+ break;
+ if (ii == wordEnd) // matched at j
+ break;
+
+ // if there's no '*' before word, then the string must
+ // match at j, but it didn't
+ if (wordStart == i)
+ return false;
+ }
+
+ // failed to locate the word
+ if (!(j < lastPossible))
+ return false;
+
+ // proceed to the next word
+ i = ii;
+ j = jj;
+ }
+
+ // pattern ended, if string also ended, then it's a match;
+ // otherwise it's a mismatch
+ return j == slen;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/daf359d3/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index b336e4b..bbc9448 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -49,6 +49,22 @@ public class StringUtil {
return (String[]) whatsLeft.toArray(new String[whatsLeft.size()]);
}
+ /**
+ * Returns a substring by removing the specified suffix. If the given string
+ * does not ends with the suffix, the string is returned without change.
+ *
+ * @param str
+ * @param suffix
+ * @return
+ */
+ public static String trimSuffix(String str, String suffix) {
+ if (str.endsWith(suffix)) {
+ return str.substring(0, str.length() - suffix.length());
+ } else {
+ return str;
+ }
+ }
+
public static String join(Iterable<String> parts, String separator) {
StringBuilder buf = new StringBuilder();
for (String p : parts) {
[22/26] kylin git commit: KYLIN-2137 Failed to run mr job when user
put a kafka jar in hive's lib folder
Posted by li...@apache.org.
KYLIN-2137 Failed to run mr job when user put a kafka jar in hive's lib folder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8314e744
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8314e744
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8314e744
Branch: refs/heads/KYLIN-1971
Commit: 8314e744f40a4042f4652502c621cc30591b7fd1
Parents: a0cfaa1
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 28 18:15:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 28 21:57:19 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/StringUtil.java | 13 +++++++++
.../engine/mr/common/AbstractHadoopJob.java | 29 ++++++--------------
2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 15872aa..b336e4b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -124,4 +124,17 @@ public class StringUtil {
return result;
}
+ public static void appendWithSeparator(StringBuilder src, String append) {
+ if (src == null) {
+ throw new IllegalArgumentException();
+ }
+ if (src.length() > 0 && src.toString().endsWith(",") == false) {
+ src.append(",");
+ }
+
+ if (StringUtils.isBlank(append) == false) {
+ src.append(append);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8314e744/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index a5b2d2e..417d63d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -61,6 +61,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.HadoopUtil;
@@ -202,24 +203,23 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String filteredHive = filterKylinHiveDependency(kylinHiveDependency);
logger.info("Hive Dependencies After Filtered: " + filteredHive);
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(filteredHive);
+ StringUtil.appendWithSeparator(kylinDependency, filteredHive);
} else {
logger.info("No hive dependency jars set in the environment, will find them from jvm:");
try {
String hiveExecJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.ql.Driver"));
- kylinDependency.append(hiveExecJarPath).append(",");
+
+ StringUtil.appendWithSeparator(kylinDependency, hiveExecJarPath);
logger.info("hive-exec jar file: " + hiveExecJarPath);
String hiveHCatJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hive.hcatalog.mapreduce.HCatInputFormat"));
- kylinDependency.append(hiveHCatJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveHCatJarPath);
logger.info("hive-catalog jar file: " + hiveHCatJarPath);
String hiveMetaStoreJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.hadoop.hive.metastore.api.Table"));
- kylinDependency.append(hiveMetaStoreJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, hiveMetaStoreJarPath);
logger.info("hive-metastore jar file: " + hiveMetaStoreJarPath);
} catch (ClassNotFoundException e) {
logger.error("Cannot found hive dependency jars: " + e);
@@ -229,19 +229,13 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for kafka dependencies
if (kylinKafkaDependency != null) {
kylinKafkaDependency = kylinKafkaDependency.replace(":", ",");
-
logger.info("Kafka Dependencies Before Filtered: " + kylinKafkaDependency);
-
- if (kylinDependency.length() > 0)
- kylinDependency.append(",");
- kylinDependency.append(kylinKafkaDependency);
+ StringUtil.appendWithSeparator(kylinDependency, kylinKafkaDependency);
} else {
-
logger.info("No Kafka dependency jars set in the environment, will find them from jvm:");
-
try {
String kafkaClientJarPath = ClassUtil.findContainingJar(Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer"));
- kylinDependency.append(kafkaClientJarPath).append(",");
+ StringUtil.appendWithSeparator(kylinDependency, kafkaClientJarPath);
logger.info("kafka jar file: " + kafkaClientJarPath);
} catch (ClassNotFoundException e) {
@@ -251,12 +245,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
// for KylinJobMRLibDir
String mrLibDir = kylinConf.getKylinJobMRLibDir();
- if (!StringUtils.isBlank(mrLibDir)) {
- if (kylinDependency.length() > 0) {
- kylinDependency.append(",");
- }
- kylinDependency.append(mrLibDir);
- }
+ StringUtil.appendWithSeparator(kylinDependency, mrLibDir);
setJobTmpJarsAndFiles(job, kylinDependency.toString());
[09/26] kylin git commit: KYLIN-1726 update sampleProducer and
kylin.sh
Posted by li...@apache.org.
KYLIN-1726 update sampleProducer and kylin.sh
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7a793e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7a793e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7a793e5c
Branch: refs/heads/KYLIN-1971
Commit: 7a793e5c33f1f7bffd5a53d64ec92065abd5856a
Parents: c804dc8
Author: shaofengshi <sh...@apache.org>
Authored: Tue Oct 25 18:19:40 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 27 09:10:16 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 7 +++--
.../source/kafka/util/KafkaSampleProducer.java | 29 ++++++++++++--------
2 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 9286055e..ad3a952 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -32,7 +32,6 @@ function retrieveDependency() {
#retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
source ${dir}/find-hbase-dependency.sh
- #source ${dir}/find-kafka-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
@@ -41,7 +40,11 @@ function retrieveDependency() {
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
- #export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+ if [ -n "$KAFKA_HOME" ]
+ then
+ source ${dir}/find-kafka-dependency.sh
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${kafka_dependency}
+ fi
}
# start command
http://git-wip-us.apache.org/repos/asf/kylin/blob/7a793e5c/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
index 3d26d3d..b8f98aa 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaSampleProducer.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
+import java.util.UUID;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
@@ -48,7 +49,6 @@ public class KafkaSampleProducer {
@SuppressWarnings("static-access")
private static final Option OPTION_TOPIC = OptionBuilder.withArgName("topic").hasArg().isRequired(true).withDescription("Kafka topic").create("topic");
private static final Option OPTION_BROKER = OptionBuilder.withArgName("broker").hasArg().isRequired(true).withDescription("Kafka broker").create("broker");
- private static final Option OPTION_DELAY = OptionBuilder.withArgName("delay").hasArg().isRequired(false).withDescription("Simulated message delay in mili-seconds, default 0").create("delay");
private static final Option OPTION_INTERVAL = OptionBuilder.withArgName("interval").hasArg().isRequired(false).withDescription("Simulated message interval in mili-seconds, default 1000").create("interval");
private static final ObjectMapper mapper = new ObjectMapper();
@@ -60,21 +60,14 @@ public class KafkaSampleProducer {
String topic, broker;
options.addOption(OPTION_TOPIC);
options.addOption(OPTION_BROKER);
- options.addOption(OPTION_DELAY);
- options.addOption(OPTION_INTERVAL);
optionsHelper.parseOptions(options, args);
logger.info("options: '" + optionsHelper.getOptionsAsString() + "'");
topic = optionsHelper.getOptionValue(OPTION_TOPIC);
broker = optionsHelper.getOptionValue(OPTION_BROKER);
- long delay = 0;
- String delayString = optionsHelper.getOptionValue(OPTION_DELAY);
- if (delayString != null) {
- delay = Long.parseLong(delayString);
- }
- long interval = 1000;
+ long interval = 10;
String intervalString = optionsHelper.getOptionValue(OPTION_INTERVAL);
if (intervalString != null) {
interval = Long.parseLong(intervalString);
@@ -101,6 +94,10 @@ public class KafkaSampleProducer {
devices.add("Andriod");
devices.add("Other");
+ List<String> genders = new ArrayList();
+ genders.add("Male");
+ genders.add("Female");
+
Properties props = new Properties();
props.put("bootstrap.servers", broker);
props.put("acks", "all");
@@ -117,15 +114,23 @@ public class KafkaSampleProducer {
Random rnd = new Random();
Map<String, Object> record = new HashMap();
while (alive == true) {
- record.put("order_time", (new Date().getTime() - delay));
+ //add normal record
+ record.put("order_time", (new Date().getTime()));
record.put("country", countries.get(rnd.nextInt(countries.size())));
record.put("category", category.get(rnd.nextInt(category.size())));
record.put("device", devices.get(rnd.nextInt(devices.size())));
record.put("qty", rnd.nextInt(10));
record.put("currency", "USD");
record.put("amount", rnd.nextDouble() * 100);
- ProducerRecord<String, String> data = new ProducerRecord<String, String>(topic, System.currentTimeMillis() + "", mapper.writeValueAsString(record));
- System.out.println("Sending 1 message");
+ //add embedded record
+ Map<String, Object> user = new HashMap();
+ user.put("id", UUID.randomUUID().toString());
+ user.put("gender", genders.get(rnd.nextInt(2)));
+ user.put("age", rnd.nextInt(20) + 10);
+ record.put("user", user);
+ //send message
+ ProducerRecord<String, String> data = new ProducerRecord<>(topic, System.currentTimeMillis() + "", mapper.writeValueAsString(record));
+ System.out.println("Sending 1 message: " + record.toString());
producer.send(data);
Thread.sleep(interval);
}
[07/26] kylin git commit: improve integer type upgrade logic
Posted by li...@apache.org.
improve integer type upgrade logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1429da5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1429da5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1429da5f
Branch: refs/heads/KYLIN-1971
Commit: 1429da5f650852402d1494785368e9175f13c379
Parents: 615e21d
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Sep 30 18:31:04 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../filter/EvaluatableFunctionTupleFilter.java | 151 -------------------
.../metadata/filter/TupleFilterSerializer.java | 20 ++-
.../apache/kylin/metadata/model/ColumnDesc.java | 20 ++-
.../apache/kylin/metadata/tuple/TupleInfo.java | 4 +-
.../apache/kylin/query/schema/OLAPTable.java | 31 +++-
5 files changed, 63 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
deleted file mode 100644
index ff24172..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableFunctionTupleFilter.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.lang.reflect.InvocationTargetException;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-
-import com.google.common.collect.Lists;
-
-public class EvaluatableFunctionTupleFilter extends BuiltInFunctionTupleFilter {
-
- private boolean constantsInitted = false;
-
- //about non-like
- private List<Object> values;
- private Object tupleValue;
-
- public EvaluatableFunctionTupleFilter(String name) {
- super(name, FilterOperatorEnum.EVAL_FUNC);
- values = Lists.newArrayListWithCapacity(1);
- values.add(null);
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
-
- // extract tuple value
- Object tupleValue = null;
- for (TupleFilter filter : this.children) {
- if (!isConstant(filter)) {
- filter.evaluate(tuple, cs);
- tupleValue = filter.getValues().iterator().next();
- }
- }
-
- TblColRef tblColRef = this.getColumn();
- DataType strDataType = DataType.getType("string");
- if (tblColRef.getType() != strDataType) {
- throw new IllegalStateException("Only String type is allow in BuiltInFunction");
- }
- ByteArray valueByteArray = (ByteArray) tupleValue;
- StringSerializer serializer = new StringSerializer(strDataType);
- String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
-
- try {
- if (isLikeFunction()) {
- return (Boolean) invokeFunction(value);
- } else {
- this.tupleValue = invokeFunction(value);
- //convert back to ByteArray format because the outer EvaluatableFunctionTupleFilter assumes input as ByteArray
- ByteBuffer buffer = ByteBuffer.allocate(valueByteArray.length() * 2);
- serializer.serialize((String) this.tupleValue, buffer);
- this.tupleValue = new ByteArray(buffer.array(), 0, buffer.position());
-
- return true;
- }
- } catch (InvocationTargetException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Collection<?> getValues() {
- this.values.set(0, tupleValue);
- return values;
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- if (!isValid()) {
- throw new IllegalStateException("must be valid");
- }
- BytesUtil.writeUTFString(name, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
- }
-
- @Override
- public boolean isEvaluable() {
- return true;
- }
-
- private boolean isConstant(TupleFilter filter) {
- return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
- }
-
- @Override
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (isLikeFunction())
- initConstants();
- return super.invokeFunction(input);
- }
-
- private void initConstants() {
- if (constantsInitted) {
- return;
- }
- //will replace the ByteArray pattern to String type
- ByteArray byteArray = (ByteArray) methodParams.get(constantPosition);
- StringSerializer s = new StringSerializer(DataType.getType("string"));
- String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
- //TODO
- //pattern = pattern.toLowerCase();//to remove upper case
- methodParams.set(constantPosition, pattern);
- constantsInitted = true;
- }
-
- //even for "tolower(s)/toupper(s)/substring(like) like pattern", the like pattern can be used for index searching
- public String getLikePattern() {
- if (!isLikeFunction()) {
- return null;
- }
-
- initConstants();
- return (String) methodParams.get(1);
- }
-
- public boolean isLikeFunction() {
- return "like".equalsIgnoreCase(this.getName());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index a051ea9..2df474e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.lang.reflect.InvocationTargetException;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.HashMap;
@@ -29,6 +30,8 @@ import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Maps;
+
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
*
@@ -45,6 +48,7 @@ public class TupleFilterSerializer {
private static final int BUFFER_SIZE = 65536;
private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>();
+ protected static final Map<TupleFilter.FilterOperatorEnum, Class> extendedTupleFilters = Maps.newHashMap();
static {
for (TupleFilter.FilterOperatorEnum op : TupleFilter.FilterOperatorEnum.values()) {
@@ -191,14 +195,20 @@ public class TupleFilterSerializer {
case UNSUPPORTED:
filter = new UnsupportedTupleFilter(op);
break;
- case EVAL_FUNC:
- filter = new EvaluatableFunctionTupleFilter(null);
- break;
case MASSIN:
filter = new MassInTupleFilter();
break;
- default:
- throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+ default: {
+ if (extendedTupleFilters.containsKey(op)) {
+ try {
+ filter = (TupleFilter) extendedTupleFilters.get(op).getConstructor().newInstance();
+ } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new IllegalStateException("Error FilterOperatorEnum: " + op.getValue());
+ }
+ }
}
return filter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 3bf0de9..a13bd37 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
/**
* Column Metadata from Source. All name should be uppercase.
@@ -47,6 +48,7 @@ public class ColumnDesc implements Serializable {
// parsed from data type
private DataType type;
+ private DataType upgradedType;
private TableDesc table;
private int zeroBasedIndex = -1;
@@ -80,6 +82,18 @@ public class ColumnDesc implements Serializable {
type = DataType.getType(datatype);
}
+ public void setUpgradedType(String datatype) {
+ this.upgradedType = DataType.getType(datatype);
+ }
+
+ public DataType getUpgradedType() {
+ if (this.upgradedType == null) {
+ return this.type;
+ } else {
+ return this.upgradedType;
+ }
+ }
+
public String getId() {
return id;
}
@@ -171,19 +185,19 @@ public class ColumnDesc implements Serializable {
if (getClass() != obj.getClass())
return false;
ColumnDesc other = (ColumnDesc) obj;
-
+
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
-
+
if (datatype == null) {
if (other.datatype != null)
return false;
} else if (!datatype.equals(other.datatype))
return false;
-
+
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
index f7c3b57..8970124 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java
@@ -94,9 +94,9 @@ public class TupleInfo {
columns.add(index, col);
if (dataTypeNames.size() > index)
- dataTypeNames.set(index, col.getType().getName());
+ dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName());
else
- dataTypeNames.add(index, col.getType().getName());
+ dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName());
}
public List<String> getAllFields() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1429da5f/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 4994433..ac70716 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -43,15 +43,21 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.enumerator.OLAPQuery.EnumeratorTypeEnum;
import org.apache.kylin.query.relnode.OLAPTableScan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -61,6 +67,8 @@ import com.google.common.collect.Sets;
*/
public class OLAPTable extends AbstractQueryableTable implements TranslatableTable {
+ protected static final Logger logger = LoggerFactory.getLogger(OLAPTable.class);
+
private static Map<String, SqlTypeName> SQLTYPE_MAPPING = new HashMap<String, SqlTypeName>();
static {
@@ -123,7 +131,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : exposedColumns) {
- RelDataType sqlType = createSqlType(typeFactory, column.getType(), column.isNullable());
+ RelDataType sqlType = createSqlType(typeFactory, column.getUpgradedType(), column.isNullable());
sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory);
fieldInfo.add(column.getName(), sqlType);
}
@@ -176,6 +184,8 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
//if exist sum(x), where x is integer/short/byte
//to avoid overflow we upgrade x's type to long
+ //this includes checking two parts:
+ //1. sum measures in cubes:
HashSet<ColumnDesc> updateColumns = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
@@ -187,9 +197,26 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
}
}
}
+ //2. All integer measures in non-cube realizations
+ MetadataManager metadataManager = MetadataManager.getInstance(olapSchema.getConfig());
+ for (IRealization realization : mgr.listAllRealizations(olapSchema.getProjectName())) {
+ if (realization.getType() == RealizationType.INVERTED_INDEX && sourceTable.getIdentity().equalsIgnoreCase(realization.getFactTable())) {
+ DataModelDesc dataModelDesc = realization.getDataModelDesc();
+ for (String metricColumn : dataModelDesc.getMetrics()) {
+ ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getFactTable() + "." + metricColumn);
+ if (columnDesc.getType().isIntegerFamily() && !columnDesc.getType().isBigInt())
+ updateColumns.add(columnDesc);
+ }
+ }
+ }
+
for (ColumnDesc upgrade : updateColumns) {
int index = tableColumns.indexOf(upgrade);
- tableColumns.get(index).setDatatype("bigint");
+ if (index < 0) {
+ throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");
+ }
+ tableColumns.get(index).setUpgradedType("bigint");
+ logger.info("To avoid overflow, upgraded {}'s type from {} to {}", tableColumns.get(index), tableColumns.get(index).getType(), tableColumns.get(index).getUpgradedType());
}
return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
[12/26] kylin git commit: Add step result check for
BuildCubeWithEngine
Posted by li...@apache.org.
Add step result check for BuildCubeWithEngine
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/71963cbe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/71963cbe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/71963cbe
Branch: refs/heads/KYLIN-1971
Commit: 71963cbeb332a4a035d83dc18a2b090b87c15d42
Parents: 3047b53
Author: shaofengshi <sh...@apache.org>
Authored: Thu Oct 27 10:48:10 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 27 10:48:10 2016 +0800
----------------------------------------------------------------------
.../kylin/provision/BuildCubeWithEngine.java | 119 ++++++++++---------
1 file changed, 61 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/71963cbe/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index f6c8801..3ae9bac 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -149,8 +149,7 @@ public class BuildCubeWithEngine {
}
-
- public void after(){
+ public void after() {
DefaultScheduler.destroyInstance();
}
@@ -167,11 +166,11 @@ public class BuildCubeWithEngine {
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f);
}
- protected void waitForJob(String jobId) {
+ protected ExecutableState waitForJob(String jobId) {
while (true) {
AbstractExecutable job = jobService.getJob(jobId);
if (job.getStatus() == ExecutableState.SUCCEED || job.getStatus() == ExecutableState.ERROR) {
- break;
+ return job.getStatus();
} else {
try {
Thread.sleep(5000);
@@ -200,17 +199,17 @@ public class BuildCubeWithEngine {
private void runTestAndAssertSucceed(String[] testCase) throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
- List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
+ List<Future<Boolean>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
for (int i = 0; i < testCase.length; i++) {
tasks.add(executorService.submit(new TestCallable(testCase[i], countDownLatch)));
}
countDownLatch.await();
try {
for (int i = 0; i < tasks.size(); ++i) {
- Future<List<String>> task = tasks.get(i);
- final List<String> jobIds = task.get();
- for (String jobId : jobIds) {
- assertJobSucceed(jobId);
+ Future<Boolean> task = tasks.get(i);
+ final Boolean result = task.get();
+ if (result == false) {
+ throw new RuntimeException("The test '" + testCase[i] + "' is failed.");
}
}
} catch (Exception ex) {
@@ -219,13 +218,7 @@ public class BuildCubeWithEngine {
}
}
- private void assertJobSucceed(String jobId) {
- if (jobService.getOutput(jobId).getState() != ExecutableState.SUCCEED) {
- throw new RuntimeException("The job '" + jobId + "' is failed.");
- }
- }
-
- private class TestCallable implements Callable<List<String>> {
+ private class TestCallable implements Callable<Boolean> {
private final String methodName;
private final CountDownLatch countDownLatch;
@@ -237,11 +230,11 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unchecked")
@Override
- public List<String> call() throws Exception {
+ public Boolean call() throws Exception {
try {
final Method method = BuildCubeWithEngine.class.getDeclaredMethod(methodName);
method.setAccessible(true);
- return (List<String>) method.invoke(BuildCubeWithEngine.this);
+ return (Boolean) method.invoke(BuildCubeWithEngine.this);
} catch (Exception e) {
logger.error(e.getMessage());
throw e;
@@ -251,9 +244,13 @@ public class BuildCubeWithEngine {
}
}
+ private void assertJobSuccess() {
+
+ }
+
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithSlr() throws Exception {
+ private boolean testInnerJoinCubeWithSlr() throws Exception {
final String cubeName = "test_kylin_cube_with_slr_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
@@ -264,17 +261,18 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date3));
+ return buildSegment(cubeName, date1, date3);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));//empty segment
+ if (buildSegment(cubeName, date1, date2) == true) {
+ return buildSegment(cubeName, date2, date3);//empty segment
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithoutSlr() throws Exception {
+ private Boolean testInnerJoinCubeWithoutSlr() throws Exception {
final String cubeName = "test_kylin_cube_without_slr_empty";
clearSegment(cubeName);
@@ -287,20 +285,23 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) {
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithoutSlr() throws Exception {
+ private boolean testLeftJoinCubeWithoutSlr() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -313,21 +314,24 @@ public class BuildCubeWithEngine {
long date4 = f.parse("2023-01-01").getTime();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));//empty segment
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) { //empty segment
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithView() throws Exception {
+ private boolean testLeftJoinCubeWithView() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -337,15 +341,13 @@ public class BuildCubeWithEngine {
long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date4 = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, date1, date4));
-
- return result;
+ return buildSegment(cubeName, date1, date4);
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testInnerJoinCubeWithView() throws Exception {
+ private boolean testInnerJoinCubeWithView() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
@@ -355,15 +357,13 @@ public class BuildCubeWithEngine {
long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date4 = f.parse("2023-01-01").getTime();
- result.add(buildSegment(cubeName, date1, date4));
-
- return result;
+ return buildSegment(cubeName, date1, date4);
}
@SuppressWarnings("unused")
// called by reflection
- private List<String> testLeftJoinCubeWithSlr() throws Exception {
+ private boolean testLeftJoinCubeWithSlr() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
@@ -376,14 +376,17 @@ public class BuildCubeWithEngine {
List<String> result = Lists.newArrayList();
if (fastBuildMode) {
- result.add(buildSegment(cubeName, date1, date4));
+ return buildSegment(cubeName, date1, date4);
} else {
- result.add(buildSegment(cubeName, date1, date2));
- result.add(buildSegment(cubeName, date2, date3));
- result.add(buildSegment(cubeName, date3, date4));
- result.add(mergeSegment(cubeName, date1, date3));//don't merge all segments
+ if (buildSegment(cubeName, date1, date2) == true) {
+ if (buildSegment(cubeName, date2, date3) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) {
+ return mergeSegment(cubeName, date1, date3);//don't merge all segments
+ }
+ }
+ }
}
- return result;
+ return false;
}
@@ -395,24 +398,24 @@ public class BuildCubeWithEngine {
cubeManager.updateCube(cubeBuilder);
}
- private String mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
+ private Boolean mergeSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.mergeSegments(cubeManager.getCube(cubeName), startDate, endDate, 0, 0, true);
DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(segment, "TEST");
jobService.addJob(job);
- waitForJob(job.getId());
- return job.getId();
+ ExecutableState state = waitForJob(job.getId());
+ return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
- private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
+ private Boolean buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegment(cubeManager.getCube(cubeName), 0, endDate);
DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
jobService.addJob(job);
- waitForJob(job.getId());
+ ExecutableState state = waitForJob(job.getId());
// if (segment.getCubeDesc().getEngineType() == IEngineAware.ID_MR_V1
// || segment.getCubeDesc().getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
// checkHFilesInHBase(segment);
// }
- return job.getId();
+ return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
@SuppressWarnings("unused")
[20/26] kylin git commit: minor, add more logging
Posted by li...@apache.org.
minor, add more logging
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59228300
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59228300
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59228300
Branch: refs/heads/KYLIN-1971
Commit: 59228300fd5f60f415e5430859c111ab39715b55
Parents: d06a425
Author: lidongsjtu <li...@apache.org>
Authored: Fri Oct 28 14:07:39 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Oct 28 14:07:46 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/59228300/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index fa23be1..b96e952 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -103,7 +103,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
dimensionsD.addAll(groupsD);
dimensionsD.addAll(otherDimsD);
Cuboid cuboid = Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
- logger.info("Cuboid identified: cube={}, cuboidId={}", cubeInstance.getName(), cuboid.getId());
+ logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, otherDimsD={}", cubeInstance.getName(), cuboid.getId(), groupsD, otherDimsD);
context.setCuboid(cuboid);
// isExactAggregation? meaning: tuples returned from storage requires no further aggregation in query engine
[14/26] kylin git commit: KYLIN-2105 remove QueryIdGenerator and
QueryIdGeneratorTest, simply use UUID as QueryId
Posted by li...@apache.org.
KYLIN-2105 remove QueryIdGenerator and QueryIdGeneratorTest, simply use UUID as QueryId
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1c24a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1c24a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1c24a9
Branch: refs/heads/KYLIN-1971
Commit: 5c1c24a9f7cc5635e0fde5eb89be595f6a07ea25
Parents: 4b3047a
Author: gaodayue <ga...@meituan.com>
Authored: Thu Oct 27 12:14:49 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Oct 27 12:16:36 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/QueryService.java | 8 +-
.../kylin/rest/util/QueryIdGenerator.java | 47 --------
.../kylin/rest/util/QueryIdGeneratorTest.java | 108 -------------------
.../coprocessor/endpoint/CubeVisitService.java | 2 +-
4 files changed, 4 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 94f2dd5..81af044 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import javax.annotation.PostConstruct;
import javax.sql.DataSource;
@@ -72,7 +73,6 @@ import org.apache.kylin.rest.model.TableMeta;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.util.QueryIdGenerator;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
import org.apache.kylin.rest.util.TableauInterceptor;
@@ -117,8 +117,6 @@ public class QueryService extends BasicService {
private final String hbaseUrl;
private final String userTableName;
- private QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
-
@Autowired
private CacheManager cacheManager;
@@ -325,7 +323,7 @@ public class QueryService extends BasicService {
throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
}
- final String queryId = queryIdGenerator.nextId(sqlRequest.getProject());
+ final String queryId = UUID.randomUUID().toString();
Map<String, String> toggles = new HashMap<>();
toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
@@ -334,7 +332,7 @@ public class QueryService extends BasicService {
}
BackdoorToggles.setToggles(toggles);
- try (SetThreadName ignored = new SetThreadName("Query-%s", queryId)) {
+ try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
String sql = sqlRequest.getSql();
String project = sqlRequest.getProject();
logger.info("Using project: " + project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
deleted file mode 100644
index 2dd19c2..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryIdGenerator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.apache.commons.lang3.time.FastDateFormat;
-
-import javax.annotation.Nonnull;
-import javax.annotation.concurrent.ThreadSafe;
-import java.util.concurrent.ThreadLocalRandom;
-
-@ThreadSafe
-public class QueryIdGenerator {
- private static final char[] base26 = "abcdefghijklmnopqrstuvwxyz".toCharArray();
- private static final FastDateFormat dateFormat = FastDateFormat.getInstance("yyyyMMdd_HHmmss");
-
- /**
- * @param project name of the project
- * @return the next query id. We try to generate unique id as much as possible, but don't guarantee it.
- */
- @Nonnull
- public String nextId(final String project) {
- char[] postfix = new char[6];
- for (int i = 0; i < postfix.length; i++) {
- postfix[i] = base26[ThreadLocalRandom.current().nextInt(base26.length)];
- }
-
- return new String(postfix);
- //return String.format("%s_%s_%s", dateFormat.format(System.currentTimeMillis()), project, new String(postfix));
- //disabled testcase: org.apache.kylin.rest.util.QueryIdGeneratorTest.testIdFormat()
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
----------------------------------------------------------------------
diff --git a/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java b/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
deleted file mode 100644
index dc82c17..0000000
--- a/server-base/src/test/java/org/apache/kylin/rest/util/QueryIdGeneratorTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.util;
-
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-public class QueryIdGeneratorTest {
-
- @Ignore
- @Test
- public void testIdFormat() {
- QueryIdGenerator generator = new QueryIdGenerator();
- for (int i = 0; i < 100; i++) {
- String queryId = generator.nextId("project");
- Assert.assertTrue(queryId.contains("project"));
- }
- }
-
- @Test
- public void testIdUniqueness() {
- QueryIdGenerator generator = new QueryIdGenerator();
- Set<String> idSet = new HashSet<>();
-
- for (int i = 0; i < 1000; i++) {
- idSet.add(generator.nextId("test"));
- }
-
- Assert.assertEquals(1000, idSet.size());
- }
-
- @Test
- public void testSingleThreadThroughput() {
- int N = 1_000_000;
- long millis = new GenIdTask(new QueryIdGenerator(), N).call();
-
- // ops / second
- double throughput = (N * 1000.0) / millis;
- System.out.format("QueryIdGenerator single thread throughput: %d ops/second\n", (int) throughput);
- }
-
- @Test
- public void testMultiThreadsThroughput() throws ExecutionException, InterruptedException {
- QueryIdGenerator generator = new QueryIdGenerator();
- int N = 1_000_000;
-
- final int numThreads = 4;
- ExecutorService pool = Executors.newFixedThreadPool(numThreads);
- Future[] futures = new Future[numThreads];
-
- for (int i = 0; i < numThreads; i++) {
- futures[i] = pool.submit(new GenIdTask(generator, N));
- }
-
- long sumMillis = 0;
- for (int i = 0; i < numThreads; i++) {
- sumMillis += (long) futures[i].get();
- }
- pool.shutdown();
-
- double avgThroughputPerThread = (N * 1000.0) / (sumMillis / (double) numThreads);
- System.out.format("QueryIdGenerator multi threads throughput: %d ops/second\n", (int) avgThroughputPerThread);
- }
-
- private static class GenIdTask implements Callable<Long> {
- private final QueryIdGenerator generator;
- private final int N;
-
- GenIdTask(QueryIdGenerator generator, int N) {
- this.generator = generator;
- this.N = N;
- }
-
- @Override
- public Long call() {
- long start = System.currentTimeMillis();
- for (int i = 0; i < N; i++) {
- generator.nextId("test");
- }
- return System.currentTimeMillis() - start;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1c24a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 506778c..da9c932 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -176,7 +176,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
String debugGitTag = "";
String queryId = request.hasQueryId() ? request.getQueryId() : "UnknownId";
- try (SetThreadName ignored = new SetThreadName("Kylin Query-%s", queryId)) {
+ try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
region = env.getRegion();
[06/26] kylin git commit: KYLIN-2030 enhancement
Posted by li...@apache.org.
KYLIN-2030 enhancement
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46284bdc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46284bdc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46284bdc
Branch: refs/heads/KYLIN-1971
Commit: 46284bdca211d59f4a3274f5579dc8fe8a48cc6e
Parents: 1429da5
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Oct 8 16:14:26 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 43 +++++++++++++++-----
.../org/apache/kylin/cube/model/CubeDesc.java | 6 +--
.../kylin/metadata/model/FunctionDesc.java | 1 +
.../test/resources/query/sql_lookup/query06.sql | 1 +
.../test/resources/query/sql_lookup/query07.sql | 1 +
5 files changed, 39 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index c8460f0..ee21b1c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -25,18 +25,22 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,20 +70,39 @@ public class CubeCapabilityChecker {
Collection<FunctionDesc> unmatchedAggregations = unmatchedAggregations(aggrFunctions, cube);
// try custom measure types
+ // in RAW query, unmatchedDimensions and unmatchedAggregations will null, so can't chose RAW cube well!
+ // if (!unmatchedDimensions.isEmpty() || !unmatchedAggregations.isEmpty()) {
tryCustomMeasureTypes(unmatchedDimensions, unmatchedAggregations, digest, cube, result);
+ // }
- // try dimension-as-measure
- if (!unmatchedAggregations.isEmpty()) {
- if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+ //more tricks
+ if (cube.getDescriptor().getFactTable().equals(digest.factTable)) {
+ //for query-on-facttable
+ //1. dimension as measure
+
+ if (!unmatchedAggregations.isEmpty()) {
tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
- } else {
- //deal with query on lookup table, like https://issues.apache.org/jira/browse/KYLIN-2030
- if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
- Set<TblColRef> dimCols = Sets.newHashSet(cube.getDataModelDesc().findFirstTable(digest.factTable).getColumns());
+ }
+ } else {
+ //for non query-on-facttable
+ if (cube.getSegments().get(0).getSnapshots().containsKey(digest.factTable)) {
+ TableDesc tableDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(digest.factTable);
+ Set<TblColRef> dimCols = Sets.newHashSet();
+ for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+ dimCols.add(columnDesc.getRef());
+ }
+
+ //1. dimension as measure, like max(cal_dt) or count( distinct col) from lookup
+ if (!unmatchedAggregations.isEmpty()) {
tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols);
- } else {
- logger.info("Skip tryDimensionAsMeasures because current cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable);
}
+
+ //2. more "dimensions" contributed by snapshot
+ if (!unmatchedDimensions.isEmpty()) {
+ unmatchedDimensions.removeAll(dimCols);
+ }
+ } else {
+ logger.info("cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0c0b4f3..159042d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -33,9 +33,9 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
+import java.util.Map.Entry;
import javax.annotation.Nullable;
@@ -67,9 +67,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
@@ -674,7 +674,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private boolean hasSingle(ArrayList<Set<String>> dimsList) {
boolean hasSingle = false;
for (Set<String> dims : dimsList) {
- if (dims.size() < 2)
+ if (dims.size() == 1)
hasSingle = true;
}
return hasSingle;
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index e7d5186..ae7f805 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -102,6 +102,7 @@ public class FunctionDesc {
}
public MeasureType<?> getMeasureType() {
+ //like max(cal_dt)
if (isDimensionAsMetric && !isCountDistinct()) {
return null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query06.sql b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
new file mode 100644
index 0000000..e2b8469
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query06.sql
@@ -0,0 +1 @@
+select max(cal_dt) as x from edw.test_cal_dt
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/46284bdc/kylin-it/src/test/resources/query/sql_lookup/query07.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_lookup/query07.sql b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
new file mode 100644
index 0000000..6be3439
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_lookup/query07.sql
@@ -0,0 +1 @@
+select site_id,count(*) as y,count(DISTINCT site_name) as x from edw.test_sites group by site_id
[11/26] kylin git commit: minor,
remove unused entry in kylin.properties
Posted by li...@apache.org.
minor, remove unused entry in kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3047b53d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3047b53d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3047b53d
Branch: refs/heads/KYLIN-1971
Commit: 3047b53d6fed4ae9a9f455223afc9dfefb758534
Parents: efa945e
Author: shaofengshi <sh...@apache.org>
Authored: Wed Oct 26 11:59:44 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Oct 27 09:10:31 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3047b53d/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index ed86bdb..33a4e76 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -48,9 +48,6 @@ kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
-# In seconds (2 days)
-kylin.storage.cleanup.time.threshold=172800000
-
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin
[19/26] kylin git commit: KYLIN-1971 fix ITStorageTest
Posted by li...@apache.org.
KYLIN-1971 fix ITStorageTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d06a425e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d06a425e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d06a425e
Branch: refs/heads/KYLIN-1971
Commit: d06a425e3b30b52ae895954a9faa034da1ad0e04
Parents: 3b9be19
Author: Li Yang <li...@apache.org>
Authored: Thu Oct 27 18:03:22 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Oct 27 18:03:22 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/TableRef.java | 8 +++
.../apache/kylin/storage/StorageMockUtils.java | 52 +++++++++++---------
.../kylin/storage/hbase/ITStorageTest.java | 18 +++----
3 files changed, 46 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
index 254ce18..7089eba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java
@@ -70,6 +70,8 @@ public class TableRef {
return Collections.unmodifiableCollection(columns.values());
}
+ // for test only
+ @Deprecated
public TblColRef makeFakeColumn(String name) {
ColumnDesc colDesc = new ColumnDesc();
colDesc.setName(name);
@@ -77,6 +79,12 @@ public class TableRef {
return new TblColRef(this, colDesc);
}
+ // for test only
+ @Deprecated
+ public TblColRef makeFakeColumn(ColumnDesc colDesc) {
+ return new TblColRef(this, colDesc);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o)
http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
index 456e768..0786f32 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
@@ -28,9 +28,11 @@ import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.TupleInfo;
@@ -39,7 +41,14 @@ import com.google.common.collect.ImmutableList;
/**
*/
public class StorageMockUtils {
- public static TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
+
+ final DataModelDesc model;
+
+ public StorageMockUtils(DataModelDesc model) {
+ this.model = model;
+ }
+
+ public TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
TupleInfo info = new TupleInfo();
int idx = 0;
@@ -47,34 +56,32 @@ public class StorageMockUtils {
info.setField(col.getName(), col, idx++);
}
- TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
+ TableRef sourceTable = groups.get(0).getTableRef();
for (FunctionDesc func : aggregations) {
- TblColRef col = func.newFakeRewriteColumn(sourceTable).getRef();
+ ColumnDesc colDesc = func.newFakeRewriteColumn(sourceTable.getTableDesc());
+ TblColRef col = sourceTable.makeFakeColumn(colDesc);
info.setField(col.getName(), col, idx++);
}
return info;
}
- public static List<TblColRef> buildGroups() {
+ public List<TblColRef> buildGroups() {
List<TblColRef> groups = new ArrayList<TblColRef>();
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- TblColRef c1 = TblColRef.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef c1 = model.findColumn("DEFAULT.TEST_KYLIN_FACT.CAL_DT");
groups.add(c1);
- TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
- TblColRef c2 = TblColRef.mockup(t2, 14, "META_CATEG_NAME", "string");
+ TblColRef c2 = model.findColumn("DEFAULT.TEST_CATEGORY_GROUPINGS.META_CATEG_NAME");
groups.add(c2);
return groups;
}
- public static List<FunctionDesc> buildAggregations1() {
+ public List<FunctionDesc> buildAggregations1() {
List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- TblColRef priceCol = TblColRef.mockup(t1, 7, "PRICE", "decimal(19,4)");
+ TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACTPRICE");
FunctionDesc f1 = new FunctionDesc();
f1.setExpression("SUM");
@@ -89,12 +96,11 @@ public class StorageMockUtils {
return functions;
}
- public static List<FunctionDesc> buildAggregations() {
+ public List<FunctionDesc> buildAggregations() {
List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- TblColRef priceCol = TblColRef.mockup(t1, 7, "PRICE", "decimal(19,4)");
- TblColRef sellerCol = TblColRef.mockup(t1, 9, "SELLER_ID", "bigint");
+ TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.PRICE");
+ TblColRef sellerCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.SELLER_ID");
FunctionDesc f1 = new FunctionDesc();
f1.setExpression("SUM");
@@ -119,7 +125,7 @@ public class StorageMockUtils {
return functions;
}
- public static CompareTupleFilter buildTs2010Filter(TblColRef column) {
+ public CompareTupleFilter buildTs2010Filter(TblColRef column) {
CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter1);
@@ -128,7 +134,7 @@ public class StorageMockUtils {
return compareFilter;
}
- public static CompareTupleFilter buildTs2011Filter(TblColRef column) {
+ public CompareTupleFilter buildTs2011Filter(TblColRef column) {
CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter1);
@@ -137,7 +143,7 @@ public class StorageMockUtils {
return compareFilter;
}
- public static CompareTupleFilter buildFilter1(TblColRef column) {
+ public CompareTupleFilter buildFilter1(TblColRef column) {
CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter1);
@@ -146,7 +152,7 @@ public class StorageMockUtils {
return compareFilter;
}
- public static CompareTupleFilter buildFilter2(TblColRef column) {
+ public CompareTupleFilter buildFilter2(TblColRef column) {
CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter2);
@@ -155,7 +161,7 @@ public class StorageMockUtils {
return compareFilter;
}
- public static CompareTupleFilter buildFilter3(TblColRef column) {
+ public CompareTupleFilter buildFilter3(TblColRef column) {
CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
compareFilter.addChild(columnFilter1);
@@ -164,7 +170,7 @@ public class StorageMockUtils {
return compareFilter;
}
- public static TupleFilter buildAndFilter(List<TblColRef> columns) {
+ public TupleFilter buildAndFilter(List<TblColRef> columns) {
CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
@@ -173,7 +179,7 @@ public class StorageMockUtils {
return andFilter;
}
- public static TupleFilter buildOrFilter(List<TblColRef> columns) {
+ public TupleFilter buildOrFilter(List<TblColRef> columns) {
CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d06a425e/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index c30455e..ea3cbdb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -53,6 +53,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
private IStorageQuery storageEngine;
private CubeInstance cube;
private StorageContext context;
+ private StorageMockUtils mockup;
@BeforeClass
public static void setupResource() throws Exception {
@@ -73,6 +74,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
context = new StorageContext();
context.setConnUrl(url);
+ mockup = new StorageMockUtils(cube.getDataModelDesc());
}
@After
@@ -84,20 +86,20 @@ public class ITStorageTest extends HBaseMetadataTestCase {
@Ignore
public void testScanOutOfLimit() {
context.setThreshold(1);
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ List<TblColRef> groups = mockup.buildGroups();
+ List<FunctionDesc> aggregations = mockup.buildAggregations();
search(groups, aggregations, null, context);
}
@Test
public void test01() {
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+ List<TblColRef> groups = mockup.buildGroups();
+ List<FunctionDesc> aggregations = mockup.buildAggregations();
+ TupleFilter filter = mockup.buildFilter1(groups.get(0));
int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
+ assertTrue(count >= 0);
}
/*
@@ -145,14 +147,12 @@ public class ITStorageTest extends HBaseMetadataTestCase {
ITupleIterator iterator = null;
try {
SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>());
- iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
+ iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
System.out.println("Tuple = " + tuple);
count++;
}
- } catch (Exception e) {
- e.printStackTrace();
} finally {
if (iterator != null)
iterator.close();
[03/26] kylin git commit: port KYLIN-2068 to
BeelineHiveClient(KYLIN-2125)
Posted by li...@apache.org.
port KYLIN-2068 to BeelineHiveClient(KYLIN-2125)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ca6837d0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ca6837d0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ca6837d0
Branch: refs/heads/KYLIN-1971
Commit: ca6837d004c0da31c95027b48db589238179f1ed
Parents: e2a932e
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 13:37:03 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/BeelineHiveClient.java | 10 +++++-----
.../java/org/apache/kylin/source/hive/CLIHiveClient.java | 4 ++--
.../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 +
.../java/org/apache/kylin/source/hive/HiveTableMeta.java | 6 ++++--
4 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 0fbc39b..c8d56a5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -115,7 +115,7 @@ public class BeelineHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
while (columns.next()) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12)));
}
builder.setAllColumns(allColumns);
@@ -139,7 +139,7 @@ public class BeelineHiveClient implements IHiveClient {
if ("".equals(resultSet.getString(1).trim())) {
break;
}
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim()));
}
builder.setPartitionColumns(partitionColumns);
}
@@ -205,9 +205,9 @@ public class BeelineHiveClient implements IHiveClient {
public static void main(String[] args) throws SQLException {
- //BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
- BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
- HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "events");
+ BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ //BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test001");
System.out.println(hiveTableMeta);
loader.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index ea74470..60cf47a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -89,11 +89,11 @@ public class CLIHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
for (FieldSchema fieldSchema : allFields) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), fieldSchema.getComment()));
}
if (partitionFields != null && partitionFields.size() > 0) {
for (FieldSchema fieldSchema : partitionFields) {
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType(), fieldSchema.getComment()));
}
}
builder.setAllColumns(allColumns);
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 346d278..388e72b 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -120,6 +120,7 @@ public class HiveSourceTableLoader {
cdesc.setDatatype(field.dataType);
}
cdesc.setId(String.valueOf(i + 1));
+ cdesc.setComment(field.comment);
columns.add(cdesc);
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
http://git-wip-us.apache.org/repos/asf/kylin/blob/ca6837d0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
index c2b7c96..784a0bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -24,15 +24,17 @@ class HiveTableMeta {
static class HiveTableColumnMeta {
String name;
String dataType;
+ String comment;
- public HiveTableColumnMeta(String name, String dataType) {
+ public HiveTableColumnMeta(String name, String dataType, String comment) {
this.name = name;
this.dataType = dataType;
+ this.comment = comment;
}
@Override
public String toString() {
- return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + '}';
+ return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + ", comment='" + comment + '\'' + '}';
}
}
[18/26] kylin git commit: KYLIN-2134 dont reuse Object[] returned to
calcite
Posted by li...@apache.org.
KYLIN-2134 dont reuse Object[] returned to calcite
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3b9be196
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3b9be196
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3b9be196
Branch: refs/heads/KYLIN-1971
Commit: 3b9be19634c8f295a9c7468bcdaf8ab2aad73fcb
Parents: 49260dd
Author: Li Yang <li...@apache.org>
Authored: Thu Oct 27 17:09:22 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Oct 27 17:09:22 2016 +0800
----------------------------------------------------------------------
.../kylin/storage/gtrecord/SegmentCubeTupleIterator.java | 4 ----
.../org/apache/kylin/query/enumerator/OLAPEnumerator.java | 8 ++++----
2 files changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 61267ae..00ba247 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -37,8 +37,6 @@ import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
public class SegmentCubeTupleIterator implements ITupleIterator {
private static final Logger logger = LoggerFactory.getLogger(SegmentCubeTupleIterator.class);
@@ -98,8 +96,6 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
}
GTRecord curRecord = gtItr.next();
- Preconditions.checkNotNull(cubeTupleConverter);
-
// translate into tuple
advMeasureFillers = cubeTupleConverter.translateResult(curRecord, tuple);
http://git-wip-us.apache.org/repos/asf/kylin/blob/3b9be196/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index dafbb5f..c7b3c71 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.enumerator;
+import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
@@ -77,10 +78,9 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
}
private Object[] convertCurrentRow(ITuple tuple) {
- // make sure the tuple layout is correct
- //assert tuple.getAllFields().equals(olapContext.returnTupleInfo.getAllFields());
-
- current = tuple.getAllValues();
+ // give calcite a new array every time, see details in KYLIN-2134
+ Object[] values = tuple.getAllValues();
+ current = Arrays.copyOf(values, values.length);
return current;
}
[16/26] kylin git commit: minor, drop kylin.job.step.timeout
Posted by li...@apache.org.
minor, drop kylin.job.step.timeout
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/468fe9ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/468fe9ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/468fe9ff
Branch: refs/heads/KYLIN-1971
Commit: 468fe9ffc3d1b6c6813481803dd59673f78718f1
Parents: a43729b
Author: Li Yang <li...@apache.org>
Authored: Thu Oct 27 15:01:51 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Oct 27 15:01:51 2016 +0800
----------------------------------------------------------------------
build/conf/kylin_job_conf_inmem.xml | 2 +-
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ----
.../java/org/apache/kylin/job/engine/JobEngineConfig.java | 7 -------
3 files changed, 1 insertion(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/build/conf/kylin_job_conf_inmem.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf_inmem.xml b/build/conf/kylin_job_conf_inmem.xml
index 1d349f6..d363507 100644
--- a/build/conf/kylin_job_conf_inmem.xml
+++ b/build/conf/kylin_job_conf_inmem.xml
@@ -81,7 +81,7 @@
<property>
<name>mapreduce.task.timeout</name>
- <value>3600000</value>
+ <value>7200000</value>
<description>Set task timeout to 1 hour</description>
</property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 455b691..8744296 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -411,10 +411,6 @@ abstract public class KylinConfigBase implements Serializable {
return getOptionalStringArray("kylin.job.admin.dls", null);
}
- public long getJobStepTimeout() {
- return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60)));
- }
-
public double getJobCuboidSizeRatio() {
return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", "0.25"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/468fe9ff/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 8ba1730..1e3eeaa 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -145,13 +145,6 @@ public class JobEngineConfig {
}
/**
- * @return the jobStepTimeout
- */
- public long getJobStepTimeout() {
- return config.getJobStepTimeout();
- }
-
- /**
* @return the asyncJobCheckInterval
*/
public int getAsyncJobCheckInterval() {
[13/26] kylin git commit: KYLIN-2116 Remove field delimiter in hive
flatten table ddl
Posted by li...@apache.org.
KYLIN-2116 Remove field delimiter in hive flatten table ddl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b3047a7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b3047a7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b3047a7
Branch: refs/heads/KYLIN-1971
Commit: 4b3047a767aa7a696bf663cab96c11894a1f901e
Parents: 71963cb
Author: lidongsjtu <li...@apache.org>
Authored: Wed Oct 26 22:49:19 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Oct 27 11:15:51 2016 +0800
----------------------------------------------------------------------
core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3047a7/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 3cc27ba..9c73a53 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -92,8 +92,6 @@ public class JoinedFlatTable {
ddl.append(colName(col.getCanonicalName()) + " " + getHiveDataType(col.getDatatype()) + "\n");
}
ddl.append(")" + "\n");
-
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'" + "\n");
ddl.append("STORED AS SEQUENCEFILE" + "\n");
ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n");
// ddl.append("TBLPROPERTIES ('serialization.null.format'='\\\\N')" +
[04/26] kylin git commit: KYLIN-2125 add BeelineHiveClient
Posted by li...@apache.org.
KYLIN-2125 add BeelineHiveClient
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2a932e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2a932e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2a932e7
Branch: refs/heads/KYLIN-1971
Commit: e2a932e7439d0034b26b99897d7c293cadb1b8e3
Parents: ea3d02f
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Oct 13 15:31:11 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/TableController.java | 9 +-
source-hive/pom.xml | 5 +
.../kylin/source/hive/BeelineHiveClient.java | 214 +++++++++++++++++++
.../source/hive/BeelineOptionsProcessor.java | 47 ++++
.../apache/kylin/source/hive/CLIHiveClient.java | 169 +++++++++++++++
.../apache/kylin/source/hive/HiveClient.java | 170 ---------------
.../kylin/source/hive/HiveClientFactory.java | 33 +++
.../source/hive/HiveSourceTableLoader.java | 89 ++++----
.../org/apache/kylin/source/hive/HiveTable.java | 16 +-
.../apache/kylin/source/hive/HiveTableMeta.java | 71 ++++++
.../kylin/source/hive/HiveTableMetaBuilder.java | 102 +++++++++
.../apache/kylin/source/hive/HqlExecutable.java | 107 ----------
.../apache/kylin/source/hive/IHiveClient.java | 36 ++++
.../hive/BeelineOptionsProcessorTest.java | 38 ++++
14 files changed, 769 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d9050c1..47ff3fe 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -47,7 +47,8 @@ import org.apache.kylin.rest.service.KafkaConfigService;
import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
+import org.apache.kylin.source.hive.IHiveClient;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -189,7 +190,7 @@ public class TableController extends BasicController {
String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
tableName = dbTableName[0] + "." + dbTableName[1];
TableDesc desc = cubeMgmtService.getMetadataManager().getTableDesc(tableName);
- if(desc == null)
+ if (desc == null)
return false;
tableType = desc.getSourceType();
@@ -315,7 +316,7 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive", method = { RequestMethod.GET })
@ResponseBody
private static List<String> showHiveDatabases() throws IOException {
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
List<String> results = null;
try {
@@ -336,7 +337,7 @@ public class TableController extends BasicController {
@RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
@ResponseBody
private static List<String> showHiveTables(@PathVariable String database) throws IOException {
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
List<String> results = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 08019d0..16cb3b4 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -65,6 +65,11 @@
<scope>provided</scope>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<classifier>hadoop2</classifier>
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
new file mode 100644
index 0000000..0fbc39b
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+public class BeelineHiveClient implements IHiveClient {
+
+ private Connection cnct;
+ private Statement stmt;
+ private DatabaseMetaData metaData;
+
+ public BeelineHiveClient(String beelineParams) {
+ if (StringUtils.isEmpty(beelineParams)) {
+ throw new IllegalArgumentException("BeelineParames cannot be empty");
+ }
+ String[] splits = StringUtils.split(beelineParams);
+ String url = null, username = null, password = null;
+ for (int i = 0; i < splits.length; i++) {
+ if ("-u".equals(splits[i])) {
+ url = stripQuotes(splits[i + 1]);
+ }
+ if ("-n".equals(splits[i])) {
+ username = stripQuotes(splits[i + 1]);
+ }
+ if ("-p".equals(splits[i])) {
+ password = stripQuotes(splits[i + 1]);
+ }
+ }
+ this.init(url, username, password);
+ }
+
+ private void init(String url, String username, String password) {
+ try {
+ Class.forName("org.apache.hive.jdbc.HiveDriver");
+ cnct = DriverManager.getConnection(url, username, password);
+ stmt = cnct.createStatement();
+ metaData = cnct.getMetaData();
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private String stripQuotes(String input) {
+ if (input.startsWith("'") && input.endsWith("'")) {
+ return StringUtils.strip(input, "'");
+ } else if (input.startsWith("\"") && input.endsWith("\"")) {
+ return StringUtils.strip(input, "\"");
+ } else {
+ return input;
+ }
+ }
+
+ public List<String> getHiveDbNames() throws Exception {
+ List<String> ret = Lists.newArrayList();
+ ResultSet schemas = metaData.getSchemas();
+ while (schemas.next()) {
+ ret.add(String.valueOf(schemas.getObject(1)));
+ }
+ return ret;
+ }
+
+ public List<String> getHiveTableNames(String database) throws Exception {
+ List<String> ret = Lists.newArrayList();
+ ResultSet tables = metaData.getTables(null, database, null, null);
+ while (tables.next()) {
+ ret.add(String.valueOf(tables.getObject(3)));
+ }
+ return ret;
+ }
+
+ @Override
+ public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws SQLException {
+ ResultSet columns = metaData.getColumns(null, database, tableName, null);
+ HiveTableMetaBuilder builder = new HiveTableMetaBuilder();
+ builder.setTableName(tableName);
+
+ List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ while (columns.next()) {
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6)));
+ }
+ builder.setAllColumns(allColumns);
+
+ stmt.execute("use " + database);
+ ResultSet resultSet = stmt.executeQuery("describe formatted " + tableName);
+ extractHiveTableMeta(resultSet, builder);
+
+ return builder.createHiveTableMeta();
+ }
+
+ private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
+ while (resultSet.next()) {
+
+ List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+ if ("# Partition Information".equals(resultSet.getString(1).trim())) {
+ resultSet.next();
+ Preconditions.checkArgument("# col_name".equals(resultSet.getString(1).trim()));
+ resultSet.next();
+ Preconditions.checkArgument("".equals(resultSet.getString(1).trim()));
+ while (resultSet.next()) {
+ if ("".equals(resultSet.getString(1).trim())) {
+ break;
+ }
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim()));
+ }
+ builder.setPartitionColumns(partitionColumns);
+ }
+
+ if ("Owner:".equals(resultSet.getString(1).trim())) {
+ builder.setOwner(resultSet.getString(2).trim());
+ }
+ if ("LastAccessTime:".equals(resultSet.getString(1).trim())) {
+ try {
+ int i = Integer.parseInt(resultSet.getString(2).trim());
+ builder.setLastAccessTime(i);
+ } catch (NumberFormatException e) {
+ builder.setLastAccessTime(0);
+ }
+ }
+ if ("Location:".equals(resultSet.getString(1).trim())) {
+ builder.setSdLocation(resultSet.getString(2).trim());
+ }
+ if ("Table Type:".equals(resultSet.getString(1).trim())) {
+ builder.setTableType(resultSet.getString(2).trim());
+ }
+ if ("Table Parameters:".equals(resultSet.getString(1).trim())) {
+ while (resultSet.next()) {
+ if (resultSet.getString(2) == null) {
+ break;
+ }
+ if ("storage_handler".equals(resultSet.getString(2).trim())) {
+ builder.setIsNative(false);//default is true
+ }
+ if ("totalSize".equals(resultSet.getString(2).trim())) {
+ builder.setFileSize(Long.parseLong(resultSet.getString(3).trim()));//default is false
+ }
+ if ("numFiles".equals(resultSet.getString(2).trim())) {
+ builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
+ }
+ }
+ }
+ if ("InputFormat:".equals(resultSet.getString(1).trim())) {
+ builder.setSdInputFormat(resultSet.getString(2).trim());
+ }
+ if ("OutputFormat:".equals(resultSet.getString(1).trim())) {
+ builder.setSdOutputFormat(resultSet.getString(2).trim());
+ }
+ }
+ }
+
+ public void close() {
+ if (this.stmt != null) {
+ try {
+ this.stmt.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ if (this.cnct != null) {
+ try {
+ this.cnct.close();
+ } catch (SQLException e) {
+ //
+ }
+ }
+ }
+
+ public static void main(String[] args) throws SQLException {
+
+ //BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "events");
+ System.out.println(hiveTableMeta);
+ loader.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
new file mode 100644
index 0000000..68cb352
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineOptionsProcessor.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+public class BeelineOptionsProcessor {
+ private final Options options = new Options();
+
+ public BeelineOptionsProcessor() {
+
+ options.addOption(OptionBuilder.hasArg().withArgName("url").create('u'));
+ options.addOption(OptionBuilder.hasArg().withArgName("username").create('n'));
+ options.addOption(OptionBuilder.hasArg().withArgName("password").create('p'));
+
+ }
+
+ public CommandLine process(String[] argv) {
+ try {
+ return new GnuParser().parse(options, argv);
+
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
new file mode 100644
index 0000000..ea74470
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.hive;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Hive meta API client for Kylin
+ * @author shaoshi
+ *
+ */
+public class CLIHiveClient implements IHiveClient {
+
+ protected HiveConf hiveConf = null;
+ protected Driver driver = null;
+ protected HiveMetaStoreClient metaStoreClient = null;
+
+ public CLIHiveClient() {
+ hiveConf = new HiveConf(CLIHiveClient.class);
+ }
+
+ /**
+ * only used by Deploy Util
+ */
+ @Override
+ public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ CommandProcessorResponse response = getDriver().run(hql);
+ int retCode = response.getResponseCode();
+ if (retCode != 0) {
+ String err = response.getErrorMessage();
+ throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
+ }
+ }
+
+ /**
+ * only used by Deploy Util
+ */
+ @Override
+ public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ for (String sql : hqls)
+ executeHQL(sql);
+ }
+
+ @Override
+ public HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception {
+ HiveTableMetaBuilder builder = new HiveTableMetaBuilder();
+ Table table = getMetaStoreClient().getTable(database, tableName);
+
+ List<FieldSchema> allFields = getMetaStoreClient().getFields(database, tableName);
+ List<FieldSchema> partitionFields = table.getPartitionKeys();
+ if (allFields == null) {
+ allFields = Lists.newArrayList();
+ }
+ if (partitionFields != null && partitionFields.size() > 0) {
+ allFields.addAll(partitionFields);
+ }
+ List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+ for (FieldSchema fieldSchema : allFields) {
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ }
+ if (partitionFields != null && partitionFields.size() > 0) {
+ for (FieldSchema fieldSchema : partitionFields) {
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(fieldSchema.getName(), fieldSchema.getType()));
+ }
+ }
+ builder.setAllColumns(allColumns);
+ builder.setPartitionColumns(partitionColumns);
+
+ builder.setSdLocation(table.getSd().getLocation());
+ builder.setFileSize(getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE));
+ builder.setFileNum(getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES));
+ builder.setIsNative(!MetaStoreUtils.isNonNativeTable(table));
+ builder.setTableName(tableName);
+ builder.setSdInputFormat(table.getSd().getInputFormat());
+ builder.setSdOutputFormat(table.getSd().getOutputFormat());
+ builder.setOwner(table.getOwner());
+ builder.setLastAccessTime(table.getLastAccessTime());
+ builder.setTableType(table.getTableType());
+
+ return builder.createHiveTableMeta();
+ }
+
+ @Override
+ public List<String> getHiveDbNames() throws Exception {
+ return getMetaStoreClient().getAllDatabases();
+ }
+
+ @Override
+ public List<String> getHiveTableNames(String database) throws Exception {
+ return getMetaStoreClient().getAllTables(database);
+ }
+
+ private HiveMetaStoreClient getMetaStoreClient() throws Exception {
+ if (metaStoreClient == null) {
+ metaStoreClient = new HiveMetaStoreClient(hiveConf);
+ }
+ return metaStoreClient;
+ }
+
+ /**
+ * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
+ *
+ * Get basic stats of table
+ * @param table
+ * - table
+ * @param statType
+ * - type of stats
+ * @return value of stats
+ */
+ private long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
+ Map<String, String> params = table.getParameters();
+ long result = 0;
+
+ if (params != null) {
+ try {
+ result = Long.parseLong(params.get(statType));
+ } catch (NumberFormatException e) {
+ result = 0;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Get the hive ql driver to execute ddl or dml
+ * @return
+ */
+ private Driver getDriver() {
+ if (driver == null) {
+ driver = new Driver(hiveConf);
+ SessionState.start(new CliSessionState(hiveConf));
+ }
+
+ return driver;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
deleted file mode 100644
index a99b304..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClient.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-/**
- * Hive meta API client for Kylin
- * @author shaoshi
- *
- */
-public class HiveClient {
-
- protected HiveConf hiveConf = null;
- protected Driver driver = null;
- protected HiveMetaStoreClient metaStoreClient = null;
-
- public HiveClient() {
- hiveConf = new HiveConf(HiveClient.class);
- }
-
- public HiveClient(Map<String, String> configMap) {
- this();
- appendConfiguration(configMap);
- }
-
- public HiveConf getHiveConf() {
- return hiveConf;
- }
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
-
- /**
- * Append or overwrite the default hive client configuration; You need call this before invoke #executeHQL;
- * @param configMap
- */
- public void appendConfiguration(Map<String, String> configMap) {
- if (configMap != null && configMap.size() > 0) {
- for (Entry<String, String> e : configMap.entrySet()) {
- hiveConf.set(e.getKey(), e.getValue());
- }
- }
- }
-
- /**
- *
- * @param hql
- * @throws CommandNeedRetryException
- * @throws IOException
- */
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
- }
- }
-
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
- for (String sql : hqls)
- executeHQL(sql);
- }
-
- private HiveMetaStoreClient getMetaStoreClient() throws Exception {
- if (metaStoreClient == null) {
- metaStoreClient = new HiveMetaStoreClient(hiveConf);
- }
- return metaStoreClient;
- }
-
- public Table getHiveTable(String database, String tableName) throws Exception {
- return getMetaStoreClient().getTable(database, tableName);
- }
-
- public List<FieldSchema> getHiveTableFields(String database, String tableName) throws Exception {
- return getMetaStoreClient().getFields(database, tableName);
- }
-
- public String getHiveTableLocation(String database, String tableName) throws Exception {
- Table t = getHiveTable(database, tableName);
- return t.getSd().getLocation();
- }
-
- public long getFileSizeForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE);
- }
-
- public long getFileNumberForTable(Table table) {
- return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
- }
-
- public List<String> getHiveDbNames() throws Exception {
- return getMetaStoreClient().getAllDatabases();
- }
-
- public List<String> getHiveTableNames(String database) throws Exception {
- return getMetaStoreClient().getAllTables(database);
- }
-
- /**
- * COPIED FROM org.apache.hadoop.hive.ql.stats.StatsUtil for backward compatibility
- *
- * Get basic stats of table
- * @param table
- * - table
- * @param statType
- * - type of stats
- * @return value of stats
- */
- public static long getBasicStatForTable(org.apache.hadoop.hive.ql.metadata.Table table, String statType) {
- Map<String, String> params = table.getParameters();
- long result = 0;
-
- if (params != null) {
- try {
- result = Long.parseLong(params.get(statType));
- } catch (NumberFormatException e) {
- result = 0;
- }
- }
- return result;
- }
-
- public boolean isNativeTable(String database, String tableName) throws Exception {
- return !MetaStoreUtils.isNonNativeTable(getMetaStoreClient().getTable(database, tableName));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
new file mode 100644
index 0000000..8c883af
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveClientFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.kylin.common.KylinConfig;
+
+public class HiveClientFactory {
+ public static IHiveClient getHiveClient() {
+ if ("cli".equals(KylinConfig.getInstanceFromEnv().getHiveClientMode())) {
+ return new CLIHiveClient();
+ } else if ("beeline".equals(KylinConfig.getInstanceFromEnv().getHiveClientMode())) {
+ return new BeelineHiveClient(KylinConfig.getInstanceFromEnv().getHiveBeelineParams());
+ } else {
+ throw new RuntimeException("cannot recognize hive client mode");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index ec9aedb..346d278 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -25,10 +25,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
@@ -37,10 +34,8 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
/**
@@ -54,25 +49,27 @@ public class HiveSourceTableLoader {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
- public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
+ public static final String OUTPUT_SURFIX = "json";
+ public static final String TABLE_FOLDER_NAME = "table";
+ public static final String TABLE_EXD_FOLDER_NAME = "table_exd";
- SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
- for (String fullTableName : hiveTables) {
- String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
- db2tables.put(parts[0], parts[1]);
- }
+ public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
- HiveClient hiveClient = new HiveClient();
- SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config));
- for (Map.Entry<String, String> entry : db2tables.entries()) {
- SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue());
- result.raiseExceptionWhenInvalid();
+ Map<String, Set<String>> db2tables = Maps.newHashMap();
+ for (String table : hiveTables) {
+ String[] parts = HadoopUtil.parseHiveTableName(table);
+ Set<String> set = db2tables.get(parts[0]);
+ if (set == null) {
+ set = Sets.newHashSet();
+ db2tables.put(parts[0], set);
+ }
+ set.add(parts[1]);
}
// extract from hive
Set<String> loadedTables = Sets.newHashSet();
for (String database : db2tables.keySet()) {
- List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient);
+ List<String> loaded = extractHiveTables(database, db2tables.get(database), config);
loadedTables.addAll(loaded);
}
@@ -85,29 +82,19 @@ public class HiveSourceTableLoader {
metaMgr.removeTableExd(hiveTable);
}
- private static List<String> extractHiveTables(String database, Set<String> tables, HiveClient hiveClient) throws IOException {
+ private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException {
List<String> loadedTables = Lists.newArrayList();
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
for (String tableName : tables) {
- Table table = null;
- List<FieldSchema> partitionFields = null;
- List<FieldSchema> fields = null;
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ HiveTableMeta hiveTableMeta;
try {
- table = hiveClient.getHiveTable(database, tableName);
- partitionFields = table.getPartitionKeys();
- fields = hiveClient.getHiveTableFields(database, tableName);
+ hiveTableMeta = hiveClient.getHiveTableMeta(database, tableName);
} catch (Exception e) {
- e.printStackTrace();
- throw new IOException(e);
+ throw new RuntimeException("cannot get HiveTableMeta", e);
}
- if (fields != null && partitionFields != null && partitionFields.size() > 0) {
- fields.addAll(partitionFields);
- }
-
- long tableSize = hiveClient.getFileSizeForTable(table);
- long tableFileNum = hiveClient.getFileNumberForTable(table);
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);
if (tableDesc == null) {
tableDesc = new TableDesc();
@@ -116,33 +103,32 @@ public class HiveSourceTableLoader {
tableDesc.setUuid(UUID.randomUUID().toString());
tableDesc.setLastModified(0);
}
- if (table.getTableType() != null) {
- tableDesc.setTableType(table.getTableType());
+ if (hiveTableMeta.tableType != null) {
+ tableDesc.setTableType(hiveTableMeta.tableType);
}
- int columnNumber = fields.size();
+ int columnNumber = hiveTableMeta.allColumns.size();
List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);
for (int i = 0; i < columnNumber; i++) {
- FieldSchema field = fields.get(i);
+ HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(field.getName().toUpperCase());
+ cdesc.setName(field.name.toUpperCase());
// use "double" in kylin for "float"
- if ("float".equalsIgnoreCase(field.getType())) {
+ if ("float".equalsIgnoreCase(field.dataType)) {
cdesc.setDatatype("double");
} else {
- cdesc.setDatatype(field.getType());
+ cdesc.setDatatype(field.dataType);
}
cdesc.setId(String.valueOf(i + 1));
- cdesc.setComment(field.getComment());
columns.add(cdesc);
}
tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
StringBuffer partitionColumnString = new StringBuffer();
- for (int i = 0, n = partitionFields.size(); i < n; i++) {
+ for (int i = 0, n = hiveTableMeta.partitionColumns.size(); i < n; i++) {
if (i > 0)
partitionColumnString.append(", ");
- partitionColumnString.append(partitionFields.get(i).getName().toUpperCase());
+ partitionColumnString.append(hiveTableMeta.partitionColumns.get(i).name.toUpperCase());
}
Map<String, String> map = metaMgr.getTableDescExd(tableDesc.getIdentity());
@@ -150,16 +136,16 @@ public class HiveSourceTableLoader {
if (map == null) {
map = Maps.newHashMap();
}
- map.put(MetadataConstants.TABLE_EXD_TABLENAME, table.getTableName());
- map.put(MetadataConstants.TABLE_EXD_LOCATION, table.getSd().getLocation());
- map.put(MetadataConstants.TABLE_EXD_IF, table.getSd().getInputFormat());
- map.put(MetadataConstants.TABLE_EXD_OF, table.getSd().getOutputFormat());
- map.put(MetadataConstants.TABLE_EXD_OWNER, table.getOwner());
- map.put(MetadataConstants.TABLE_EXD_LAT, String.valueOf(table.getLastAccessTime()));
+ map.put(MetadataConstants.TABLE_EXD_TABLENAME, hiveTableMeta.tableName);
+ map.put(MetadataConstants.TABLE_EXD_LOCATION, hiveTableMeta.sdLocation);
+ map.put(MetadataConstants.TABLE_EXD_IF, hiveTableMeta.sdInputFormat);
+ map.put(MetadataConstants.TABLE_EXD_OF, hiveTableMeta.sdOutputFormat);
+ map.put(MetadataConstants.TABLE_EXD_OWNER, hiveTableMeta.owner);
+ map.put(MetadataConstants.TABLE_EXD_LAT, String.valueOf(hiveTableMeta.lastAccessTime));
map.put(MetadataConstants.TABLE_EXD_PC, partitionColumnString.toString());
- map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(tableSize));
- map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(tableFileNum));
- map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(partitionFields != null && partitionFields.size() > 0).toString());
+ map.put(MetadataConstants.TABLE_EXD_TFS, String.valueOf(hiveTableMeta.fileSize));
+ map.put(MetadataConstants.TABLE_EXD_TNF, String.valueOf(hiveTableMeta.fileNum));
+ map.put(MetadataConstants.TABLE_EXD_PARTITIONED, Boolean.valueOf(hiveTableMeta.partitionColumns.size() > 0).toString());
metaMgr.saveSourceTable(tableDesc);
metaMgr.saveTableExd(tableDesc.getIdentity(), map);
@@ -168,4 +154,5 @@ public class HiveSourceTableLoader {
return loadedTables;
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
index dcc43ff..97e9990 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTable.java
@@ -37,11 +37,17 @@ public class HiveTable implements ReadableTable {
final private String database;
final private String hiveTable;
- private HiveClient hiveClient;
+ private IHiveClient hiveClient;
+ private HiveTableMeta hiveTableMeta;
public HiveTable(TableDesc tableDesc) {
this.database = tableDesc.getDatabase();
this.hiveTable = tableDesc.getName();
+ try {
+ this.hiveTableMeta = getHiveClient().getHiveTableMeta(database, hiveTable);
+ } catch (Exception e) {
+ throw new RuntimeException("cannot get HiveTableMeta", e);
+ }
}
@Override
@@ -58,7 +64,7 @@ public class HiveTable implements ReadableTable {
long lastModified = sizeAndLastModified.getSecond();
// for non-native hive table, cannot rely on size & last modified on HDFS
- if (getHiveClient().isNativeTable(database, hiveTable) == false) {
+ if (this.hiveTableMeta.isNative == false) {
lastModified = System.currentTimeMillis(); // assume table is ever changing
}
@@ -80,13 +86,13 @@ public class HiveTable implements ReadableTable {
return override;
}
- return getHiveClient().getHiveTableLocation(database, hiveTable);
+ return this.hiveTableMeta.sdLocation;
}
- public HiveClient getHiveClient() {
+ public IHiveClient getHiveClient() {
if (hiveClient == null) {
- hiveClient = new HiveClient();
+ hiveClient = HiveClientFactory.getHiveClient();
}
return hiveClient;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
new file mode 100644
index 0000000..c2b7c96
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.util.List;
+
+class HiveTableMeta {
+ static class HiveTableColumnMeta {
+ String name;
+ String dataType;
+
+ public HiveTableColumnMeta(String name, String dataType) {
+ this.name = name;
+ this.dataType = dataType;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveTableColumnMeta{" + "name='" + name + '\'' + ", dataType='" + dataType + '\'' + '}';
+ }
+ }
+
+ String tableName;
+ String sdLocation;//sd is short for storage descriptor
+ String sdInputFormat;
+ String sdOutputFormat;
+ String owner;
+ String tableType;
+ int lastAccessTime;
+ long fileSize;
+ long fileNum;
+ boolean isNative;
+ List<HiveTableColumnMeta> allColumns;
+ List<HiveTableColumnMeta> partitionColumns;
+
+ public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
+ this.tableName = tableName;
+ this.sdLocation = sdLocation;
+ this.sdInputFormat = sdInputFormat;
+ this.sdOutputFormat = sdOutputFormat;
+ this.owner = owner;
+ this.tableType = tableType;
+ this.lastAccessTime = lastAccessTime;
+ this.fileSize = fileSize;
+ this.fileNum = fileNum;
+ this.isNative = isNative;
+ this.allColumns = allColumns;
+ this.partitionColumns = partitionColumns;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveTableMeta{" + "tableName='" + tableName + '\'' + ", sdLocation='" + sdLocation + '\'' + ", sdInputFormat='" + sdInputFormat + '\'' + ", sdOutputFormat='" + sdOutputFormat + '\'' + ", owner='" + owner + '\'' + ", tableType='" + tableType + '\'' + ", lastAccessTime=" + lastAccessTime + ", fileSize=" + fileSize + ", fileNum=" + fileNum + ", isNative=" + isNative + ", allColumns=" + allColumns + ", partitionColumns=" + partitionColumns + '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
new file mode 100644
index 0000000..7a3e5d6
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+public class HiveTableMetaBuilder {
+ private String tableName;
+ private String sdLocation;
+ private String sdInputFormat;
+ private String sdOutputFormat;
+ private String owner;
+ private String tableType;
+ private int lastAccessTime;
+ private long fileSize;
+ private long fileNum;
+ private boolean isNative = true;
+ private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
+ private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
+
+ public HiveTableMetaBuilder setTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdLocation(String sdLocation) {
+ this.sdLocation = sdLocation;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdInputFormat(String sdInputFormat) {
+ this.sdInputFormat = sdInputFormat;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setSdOutputFormat(String sdOutputFormat) {
+ this.sdOutputFormat = sdOutputFormat;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setOwner(String owner) {
+ this.owner = owner;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setTableType(String tableType) {
+ this.tableType = tableType;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setLastAccessTime(int lastAccessTime) {
+ this.lastAccessTime = lastAccessTime;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setFileNum(long fileNum) {
+ this.fileNum = fileNum;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setIsNative(boolean isNative) {
+ this.isNative = isNative;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setAllColumns(List<HiveTableMeta.HiveTableColumnMeta> allColumns) {
+ this.allColumns = allColumns;
+ return this;
+ }
+
+ public HiveTableMetaBuilder setPartitionColumns(List<HiveTableMeta.HiveTableColumnMeta> partitionColumns) {
+ this.partitionColumns = partitionColumns;
+ return this;
+ }
+
+ public HiveTableMeta createHiveTableMeta() {
+ return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
deleted file mode 100644
index 79493a4..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HqlExecutable.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.job.exception.ExecuteException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableContext;
-import org.apache.kylin.job.execution.ExecuteResult;
-import org.datanucleus.store.types.backed.HashMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-
-/**
- */
-public class HqlExecutable extends AbstractExecutable {
-
- private static final Logger logger = LoggerFactory.getLogger(HqlExecutable.class);
-
- private static final String HQL = "hql";
- private static final String HIVE_CONFIG = "hive-config";
-
- public HqlExecutable() {
- super();
- }
-
- @Override
- protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- try {
- Map<String, String> configMap = getConfiguration();
- HiveClient hiveClient = new HiveClient(configMap);
-
- for (String hql : getHqls()) {
- hiveClient.executeHQL(hql);
- }
- return new ExecuteResult(ExecuteResult.State.SUCCEED);
- } catch (Exception e) {
- logger.error("error run hive query:" + getHqls(), e);
- return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
- }
- }
-
- public void setConfiguration(Map<String, String> configMap) {
- if (configMap != null) {
- String configStr = "";
- try {
- configStr = JsonUtil.writeValueAsString(configMap);
- } catch (JsonProcessingException e) {
- e.printStackTrace();
- }
- setParam(HIVE_CONFIG, configStr);
- }
- }
-
- @SuppressWarnings("unchecked")
- private Map<String, String> getConfiguration() {
- String configStr = getParam(HIVE_CONFIG);
- Map<String, String> result = null;
- if (configStr != null) {
- try {
- result = JsonUtil.readValue(configStr, HashMap.class);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- return result;
- }
-
- public void setHqls(List<String> hqls) {
- setParam(HQL, StringUtils.join(hqls, ";"));
- }
-
- private List<String> getHqls() {
- final String hqls = getParam(HQL);
- if (hqls != null) {
- return Lists.newArrayList(StringUtils.split(hqls, ";"));
- } else {
- return Collections.emptyList();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
new file mode 100644
index 0000000..f218cce
--- /dev/null
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import java.io.IOException;
+import java.util.List;
+
+public interface IHiveClient {
+ void executeHQL(String hql) throws CommandNeedRetryException, IOException;
+
+ void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException;
+
+ HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception;
+
+ List<String> getHiveDbNames() throws Exception;
+
+ List<String> getHiveTableNames(String database) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2a932e7/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
new file mode 100644
index 0000000..84da0a2
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineOptionsProcessorTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class BeelineOptionsProcessorTest {
+ @Ignore
+ @Test
+ public void foo() {
+ String param = "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000'";
+ BeelineOptionsProcessor processor = new BeelineOptionsProcessor();
+ CommandLine commandLine = processor.process(StringUtils.split(param));
+ String n = commandLine.getOptionValue('n');
+ String u = commandLine.getOptionValue('u');
+ String p = commandLine.getOptionValue('p');
+
+ }
+}
[08/26] kylin git commit: port KYLIN-2012 to new interface introduced
in KYLIN-2125
Posted by li...@apache.org.
port KYLIN-2012 to new interface introduced in KYLIN-2125
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/615e21d7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/615e21d7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/615e21d7
Branch: refs/heads/KYLIN-1971
Commit: 615e21d7fb0dae651ab643949ef5078bef7b12ea
Parents: ca6837d
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Oct 26 14:04:56 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 5 ++-
.../kylin/source/hive/BeelineHiveClient.java | 2 +-
.../source/hive/HiveSourceTableLoader.java | 32 +++++++--------
.../apache/kylin/source/hive/SchemaChecker.java | 41 ++++++++------------
4 files changed, 36 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4a24ad2..54feb24 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -45,8 +45,9 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.HiveClient;
+import org.apache.kylin.source.hive.HiveClientFactory;
import org.apache.kylin.source.hive.HiveCmdBuilder;
+import org.apache.kylin.source.hive.IHiveClient;
import org.apache.kylin.source.kafka.TimedJsonStreamParser;
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
@@ -234,7 +235,7 @@ public class DeployUtil {
String tableFileDir = temp.getParent();
temp.delete();
- HiveClient hiveClient = new HiveClient();
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index c8d56a5..a84aeb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -207,7 +207,7 @@ public class BeelineHiveClient implements IHiveClient {
BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
//BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
- HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test001");
+ HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part");
System.out.println(hiveTableMeta);
loader.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 388e72b..401e720 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
@@ -34,8 +35,10 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
/**
@@ -49,27 +52,25 @@ public class HiveSourceTableLoader {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
- public static final String OUTPUT_SURFIX = "json";
- public static final String TABLE_FOLDER_NAME = "table";
- public static final String TABLE_EXD_FOLDER_NAME = "table_exd";
-
public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
- Map<String, Set<String>> db2tables = Maps.newHashMap();
- for (String table : hiveTables) {
- String[] parts = HadoopUtil.parseHiveTableName(table);
- Set<String> set = db2tables.get(parts[0]);
- if (set == null) {
- set = Sets.newHashSet();
- db2tables.put(parts[0], set);
- }
- set.add(parts[1]);
+ SetMultimap<String, String> db2tables = LinkedHashMultimap.create();
+ for (String fullTableName : hiveTables) {
+ String[] parts = HadoopUtil.parseHiveTableName(fullTableName);
+ db2tables.put(parts[0], parts[1]);
+ }
+
+ IHiveClient hiveClient = HiveClientFactory.getHiveClient();
+ SchemaChecker checker = new SchemaChecker(hiveClient, MetadataManager.getInstance(config), CubeManager.getInstance(config));
+ for (Map.Entry<String, String> entry : db2tables.entries()) {
+ SchemaChecker.CheckResult result = checker.allowReload(entry.getKey(), entry.getValue());
+ result.raiseExceptionWhenInvalid();
}
// extract from hive
Set<String> loadedTables = Sets.newHashSet();
for (String database : db2tables.keySet()) {
- List<String> loaded = extractHiveTables(database, db2tables.get(database), config);
+ List<String> loaded = extractHiveTables(database, db2tables.get(database), hiveClient);
loadedTables.addAll(loaded);
}
@@ -82,12 +83,11 @@ public class HiveSourceTableLoader {
metaMgr.removeTableExd(hiveTable);
}
- private static List<String> extractHiveTables(String database, Set<String> tables, KylinConfig config) throws IOException {
+ private static List<String> extractHiveTables(String database, Set<String> tables, IHiveClient hiveClient) throws IOException {
List<String> loadedTables = Lists.newArrayList();
MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
for (String tableName : tables) {
- IHiveClient hiveClient = HiveClientFactory.getHiveClient();
HiveTableMeta hiveTableMeta;
try {
hiveTableMeta = hiveClient.getHiveTableMeta(database, tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/615e21d7/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
index 319ebee..87a8870 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/SchemaChecker.java
@@ -27,8 +27,6 @@ import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -46,7 +44,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public class SchemaChecker {
- private final HiveClient hiveClient;
+ private final IHiveClient hiveClient;
private final MetadataManager metadataManager;
private final CubeManager cubeManager;
@@ -87,23 +85,16 @@ public class SchemaChecker {
}
}
- SchemaChecker(HiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) {
+ SchemaChecker(IHiveClient hiveClient, MetadataManager metadataManager, CubeManager cubeManager) {
this.hiveClient = checkNotNull(hiveClient, "hiveClient is null");
this.metadataManager = checkNotNull(metadataManager, "metadataManager is null");
this.cubeManager = checkNotNull(cubeManager, "cubeManager is null");
}
- private List<FieldSchema> fetchSchema(String dbName, String tblName) throws Exception {
- List<FieldSchema> fields = Lists.newArrayList();
- fields.addAll(hiveClient.getHiveTableFields(dbName, tblName));
-
- Table table = hiveClient.getHiveTable(dbName, tblName);
- List<FieldSchema> partitionFields = table.getPartitionKeys();
- if (partitionFields != null) {
- fields.addAll(partitionFields);
- }
-
- return fields;
+ private List<HiveTableMeta.HiveTableColumnMeta> fetchSchema(String dbName, String tblName) throws Exception {
+ List<HiveTableMeta.HiveTableColumnMeta> columnMetas = Lists.newArrayList();
+ columnMetas.addAll(hiveClient.getHiveTableMeta(dbName, tblName).allColumns);
+ return columnMetas;
}
private List<CubeInstance> findCubeByTable(final String fullTableName) {
@@ -128,12 +119,12 @@ public class SchemaChecker {
return ImmutableList.copyOf(relatedCubes);
}
- private boolean isColumnCompatible(ColumnDesc column, FieldSchema field) {
- if (!column.getName().equalsIgnoreCase(field.getName())) {
+ private boolean isColumnCompatible(ColumnDesc column, HiveTableMeta.HiveTableColumnMeta field) {
+ if (!column.getName().equalsIgnoreCase(field.name)) {
return false;
}
- String typeStr = field.getType();
+ String typeStr = field.dataType;
// kylin uses double internally for float, see HiveSourceTableLoader.java
// TODO should this normalization to be in DataType class ?
if ("float".equalsIgnoreCase(typeStr)) {
@@ -159,7 +150,7 @@ public class SchemaChecker {
* @param fieldsMap current hive schema of `table`
* @return true if all columns used in `cube` has compatible schema with `fieldsMap`, false otherwise
*/
- private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, FieldSchema> fieldsMap) {
+ private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc table, Map<String, HiveTableMeta.HiveTableColumnMeta> fieldsMap) {
Set<ColumnDesc> usedColumns = Sets.newHashSet();
for (TblColRef col : cube.getAllColumns()) {
usedColumns.add(col.getColumnDesc());
@@ -168,7 +159,7 @@ public class SchemaChecker {
List<String> violateColumns = Lists.newArrayList();
for (ColumnDesc column : table.getColumns()) {
if (usedColumns.contains(column)) {
- FieldSchema field = fieldsMap.get(column.getName());
+ HiveTableMeta.HiveTableColumnMeta field = fieldsMap.get(column.getName());
if (field == null || !isColumnCompatible(column, field)) {
violateColumns.add(column.getName());
}
@@ -184,7 +175,7 @@ public class SchemaChecker {
* @param fields current table metadata in hive
* @return true if only new columns are appended in hive, false otherwise
*/
- private boolean checkAllColumnsInTableDesc(TableDesc table, List<FieldSchema> fields) {
+ private boolean checkAllColumnsInTableDesc(TableDesc table, List<HiveTableMeta.HiveTableColumnMeta> fields) {
if (table.getColumnCount() > fields.size()) {
return false;
}
@@ -206,15 +197,15 @@ public class SchemaChecker {
return CheckResult.validOnFirstLoad(fullTableName);
}
- List<FieldSchema> currentFields;
- Map<String, FieldSchema> currentFieldsMap = Maps.newHashMap();
+ List<HiveTableMeta.HiveTableColumnMeta> currentFields;
+ Map<String, HiveTableMeta.HiveTableColumnMeta> currentFieldsMap = Maps.newHashMap();
try {
currentFields = fetchSchema(dbName, tblName);
} catch (Exception e) {
return CheckResult.invalidOnFetchSchema(fullTableName, e);
}
- for (FieldSchema field : currentFields) {
- currentFieldsMap.put(field.getName().toUpperCase(), field);
+ for (HiveTableMeta.HiveTableColumnMeta field : currentFields) {
+ currentFieldsMap.put(field.name.toUpperCase(), field);
}
List<String> issues = Lists.newArrayList();
[05/26] kylin git commit: KYLIN-2030 bug fix
Posted by li...@apache.org.
KYLIN-2030 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/324e4239
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/324e4239
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/324e4239
Branch: refs/heads/KYLIN-1971
Commit: 324e4239c308ba5b14a46eb3187dad3186489f92
Parents: 46284bd
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Oct 9 19:07:52 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Oct 27 08:30:13 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 23 ++++++++++----------
.../kylin/query/relnode/OLAPAggregateRel.java | 22 +++++++++++--------
2 files changed, 25 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index ee21b1c..e509d98 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -81,7 +81,7 @@ public class CubeCapabilityChecker {
//1. dimension as measure
if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
+ tryDimensionAsMeasures(unmatchedAggregations, result, cube.getDescriptor().listDimensionColumnsIncludingDerived());
}
} else {
//for non query-on-facttable
@@ -92,10 +92,18 @@ public class CubeCapabilityChecker {
dimCols.add(columnDesc.getRef());
}
- //1. dimension as measure, like max(cal_dt) or count( distinct col) from lookup
+ //1. all aggregations on lookup table can be done. For distinct count, mark them all DimensionAsMeasures
+ // so that the measure has a chance to be upgraded to DimCountDistinctMeasureType in org.apache.kylin.metadata.model.FunctionDesc#reInitMeasureType
if (!unmatchedAggregations.isEmpty()) {
- tryDimensionAsMeasures(unmatchedAggregations, digest, cube, result, dimCols);
+ Iterator<FunctionDesc> itr = unmatchedAggregations.iterator();
+ while (itr.hasNext()) {
+ FunctionDesc functionDesc = itr.next();
+ if (dimCols.containsAll(functionDesc.getParameter().getColRefs())) {
+ itr.remove();
+ }
+ }
}
+ tryDimensionAsMeasures(Lists.newArrayList(aggrFunctions), result, dimCols);
//2. more "dimensions" contributed by snapshot
if (!unmatchedDimensions.isEmpty()) {
@@ -159,19 +167,12 @@ public class CubeCapabilityChecker {
return result;
}
- private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result, Set<TblColRef> dimCols) {
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
+ private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result, Set<TblColRef> dimCols) {
Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
while (it.hasNext()) {
FunctionDesc functionDesc = it.next();
- if (cubeFuncs.contains(functionDesc)) {
- it.remove();
- continue;
- }
-
// let calcite handle count
if (functionDesc.isCount()) {
it.remove();
http://git-wip-us.apache.org/repos/asf/kylin/blob/324e4239/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d73206a..62351d3 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -296,16 +296,20 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
private void translateAggregation() {
- // now the realization is known, replace aggregations with what's defined on MeasureDesc
- List<MeasureDesc> measures = this.context.realization.getMeasures();
- List<FunctionDesc> newAggrs = Lists.newArrayList();
- for (FunctionDesc aggFunc : this.aggregations) {
- newAggrs.add(findInMeasures(aggFunc, measures));
+ if (!noPrecaculatedFieldsAvailable()) {
+ // now the realization is known, replace aggregations with what's defined on MeasureDesc
+ List<MeasureDesc> measures = this.context.realization.getMeasures();
+ List<FunctionDesc> newAggrs = Lists.newArrayList();
+ for (FunctionDesc aggFunc : this.aggregations) {
+ newAggrs.add(findInMeasures(aggFunc, measures));
+ }
+ this.aggregations.clear();
+ this.aggregations.addAll(newAggrs);
+ this.context.aggregations.clear();
+ this.context.aggregations.addAll(newAggrs);
+ } else {
+ //the realization is not contributing pre-calculated fields at all
}
- this.aggregations.clear();
- this.aggregations.addAll(newAggrs);
- this.context.aggregations.clear();
- this.context.aggregations.addAll(newAggrs);
}
private FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) {