You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/08 11:55:43 UTC
[01/35] kylin git commit: #KYLIN-490 support multiple column distinct
count [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase0.98 a5ec512be -> de4ddd143 (forced update)
#KYLIN-490 support multiple column distinct count
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88a1c71d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88a1c71d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88a1c71d
Branch: refs/heads/master-hbase0.98
Commit: 88a1c71dde855c693b230f67b92c4cd067d43b2b
Parents: f72a3f6
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Mar 22 19:22:22 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 27 15:20:40 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/ParamAsMeasureCount.java | 30 +++++
.../BitmapIntersectDistinctCountAggFunc.java | 9 +-
.../measure/percentile/PercentileAggFunc.java | 9 +-
.../kylin/metadata/model/FunctionDesc.java | 62 ++++++---
.../kylin/metadata/model/ParameterDesc.java | 135 +++++++++++++++++--
.../resources/query/sql_distinct/query08.sql | 24 ++++
.../kylin/query/relnode/OLAPAggregateRel.java | 86 +++++++++---
7 files changed, 304 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/core-metadata/src/main/java/org/apache/kylin/measure/ParamAsMeasureCount.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/ParamAsMeasureCount.java b/core-metadata/src/main/java/org/apache/kylin/measure/ParamAsMeasureCount.java
new file mode 100644
index 0000000..b9bcd10
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/ParamAsMeasureCount.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure;
+
+public interface ParamAsMeasureCount {
+ /**
+ * Get how many parameters are required to identify the measure
+ * Negative value is for var arguments function
+ * @return 0 ==> all parameters
+ * positive number ==> parameter count
+ * negative number ==> parameter count - required number
+ */
+ int getParamAsMeasureCount();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
index cd4d306..a1e2665 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
@@ -17,6 +17,8 @@
*/
package org.apache.kylin.measure.bitmap;
+import org.apache.kylin.measure.ParamAsMeasureCount;
+
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -27,9 +29,14 @@ import java.util.Map;
* Example: intersect_count(uuid, event, array['A', 'B', 'C']), meaning find the count of uuid in all A/B/C 3 bitmaps
* requires an bitmap count distinct measure of uuid, and an dimension of event
*/
-public class BitmapIntersectDistinctCountAggFunc {
+public class BitmapIntersectDistinctCountAggFunc implements ParamAsMeasureCount {
private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
+ @Override
+ public int getParamAsMeasureCount() {
+ return -2;
+ }
+
public static class RetentionPartialResult {
Map<Object, BitmapCounter> map;
List keyList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
index ad02019..d3cec8f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
@@ -18,7 +18,9 @@
package org.apache.kylin.measure.percentile;
-public class PercentileAggFunc {
+import org.apache.kylin.measure.ParamAsMeasureCount;
+
+public class PercentileAggFunc implements ParamAsMeasureCount{
public static PercentileCounter init() {
return null;
}
@@ -41,4 +43,9 @@ public class PercentileAggFunc {
public static double result(PercentileCounter counter) {
return counter == null ? 0L : counter.getResultEstimate();
}
+
+ @Override
+ public int getParamAsMeasureCount() {
+ return 1;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index cbd7574..61c5fac 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,22 +18,26 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
/**
*/
@@ -48,7 +52,7 @@ public class FunctionDesc implements Serializable {
r.returnDataType = DataType.getType(returnType);
return r;
}
-
+
public static final String FUNC_SUM = "SUM";
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
@@ -95,7 +99,7 @@ public class FunctionDesc implements Serializable {
}
}
- if(parameter != null)
+ if (parameter != null)
parameter.setColRefs(colRefs);
}
@@ -140,6 +144,8 @@ public class FunctionDesc implements Serializable {
return getParameter().getValue();
} else if (isCount()) {
return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
+ } else if (isCountDistinct()) {
+ return "_KY_" + getFullExpressionInAlphabetOrder().replaceAll("[(),. ]", "_");
} else {
return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_");
}
@@ -197,6 +203,25 @@ public class FunctionDesc implements Serializable {
return sb.toString();
}
+ /**
+ * Parameters' name appears in alphabet order.
+ * This method is used for funcs whose parameters appear in arbitrary order
+ */
+ public String getFullExpressionInAlphabetOrder() {
+ StringBuilder sb = new StringBuilder(expression);
+ sb.append("(");
+ ParameterDesc localParam = parameter;
+ List<String> flatParams = Lists.newArrayList();
+ while (localParam != null) {
+ flatParams.add(localParam.getValue());
+ localParam = localParam.getNextParameter();
+ }
+ Collections.sort(flatParams);
+ sb.append(Joiner.on(",").join(flatParams));
+ sb.append(")");
+ return sb.toString();
+ }
+
public boolean isDimensionAsMetric() {
return isDimensionAsMetric;
}
@@ -264,13 +289,20 @@ public class FunctionDesc implements Serializable {
return false;
} else if (!expression.equals(other.expression))
return false;
- // NOTE: don't check the parameter of count()
- if (isCount() == false) {
+ if (isCountDistinct()) {
+ // for count distinct func, param's order doesn't matter
+ if (parameter == null) {
+ if (other.parameter != null)
+ return false;
+ } else {
+ return parameter.equalInArbitraryOrder(other.parameter);
+ }
+ } else if (!isCount()) { // NOTE: don't check the parameter of count()
if (parameter == null) {
if (other.parameter != null)
return false;
} else {
- if (!parameter.equals(other.parameter))
+ if (!parameter.equals(other.parameter))
return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 8ad20a8..5ba2f14 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -18,17 +18,19 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-
-import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.List;
+import com.google.common.collect.Sets;
/**
*/
@@ -38,9 +40,9 @@ public class ParameterDesc implements Serializable {
public static ParameterDesc newInstance(Object... objs) {
if (objs.length == 0)
throw new IllegalArgumentException();
-
+
ParameterDesc r = new ParameterDesc();
-
+
Object obj = objs[0];
if (obj instanceof TblColRef) {
TblColRef col = (TblColRef) obj;
@@ -51,7 +53,7 @@ public class ParameterDesc implements Serializable {
r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT;
r.value = (String) obj;
}
-
+
if (objs.length >= 2) {
r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1, objs.length));
if (r.nextParameter.colRefs.size() > 0) {
@@ -63,7 +65,7 @@ public class ParameterDesc implements Serializable {
}
return r;
}
-
+
@JsonProperty("type")
private String type;
@JsonProperty("value")
@@ -74,6 +76,15 @@ public class ParameterDesc implements Serializable {
private ParameterDesc nextParameter;
private List<TblColRef> colRefs = ImmutableList.of();
+ private Set<PlainParameter> plainParameters = null;
+
+ // Lazy evaluation
+ public Set<PlainParameter> getPlainParameters() {
+ if (plainParameters == null) {
+ plainParameters = PlainParameter.createFromParameterDesc(this);
+ }
+ return plainParameters;
+ }
public String getType() {
return type;
@@ -86,7 +97,7 @@ public class ParameterDesc implements Serializable {
public String getValue() {
return value;
}
-
+
void setValue(String value) {
this.value = value;
}
@@ -94,7 +105,7 @@ public class ParameterDesc implements Serializable {
public List<TblColRef> getColRefs() {
return colRefs;
}
-
+
void setColRefs(List<TblColRef> colRefs) {
this.colRefs = colRefs;
}
@@ -118,7 +129,7 @@ public class ParameterDesc implements Serializable {
if (type != null ? !type.equals(that.type) : that.type != null)
return false;
-
+
ParameterDesc p = this, q = that;
int refi = 0, refj = 0;
for (; p != null && q != null; p = p.nextParameter, q = q.nextParameter) {
@@ -138,10 +149,24 @@ public class ParameterDesc implements Serializable {
return false;
}
}
-
+
return p == null && q == null;
}
+ public boolean equalInArbitraryOrder(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ParameterDesc that = (ParameterDesc) o;
+
+ Set<PlainParameter> thisPlainParams = this.getPlainParameters();
+ Set<PlainParameter> thatPlainParams = that.getPlainParameters();
+
+ return thisPlainParams.containsAll(thatPlainParams) && thatPlainParams.containsAll(thisPlainParams);
+ }
+
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
@@ -154,4 +179,88 @@ public class ParameterDesc implements Serializable {
return "ParameterDesc [type=" + type + ", value=" + value + ", nextParam=" + nextParameter + "]";
}
+ /**
+ * PlainParameter is created to present ParameterDesc in List style.
+ * Compared to ParameterDesc its advantage is:
+ * 1. easy to compare without considering order
+ * 2. easy to compare one by one
+ */
+ private static class PlainParameter {
+ private String type;
+ private String value;
+ private TblColRef colRef = null;
+
+ private PlainParameter() {
+ }
+
+ public boolean isColumnType() {
+ return FunctionDesc.PARAMETER_TYPE_COLUMN.equals(type);
+ }
+
+ static Set<PlainParameter> createFromParameterDesc(ParameterDesc parameterDesc) {
+ Set<PlainParameter> result = Sets.newHashSet();
+ ParameterDesc local = parameterDesc;
+ List<TblColRef> totalColRef = parameterDesc.colRefs;
+ Integer colIndex = 0;
+ while (local != null) {
+ if (local.isColumnType()) {
+ result.add(createSingleColumnParameter(local, totalColRef.get(colIndex++)));
+ } else {
+ result.add(createSingleValueParameter(local));
+ }
+ local = local.nextParameter;
+ }
+ return result;
+ }
+
+ static PlainParameter createSingleValueParameter(ParameterDesc parameterDesc) {
+ PlainParameter single = new PlainParameter();
+ single.type = parameterDesc.type;
+ single.value = parameterDesc.value;
+ return single;
+ }
+
+ static PlainParameter createSingleColumnParameter(ParameterDesc parameterDesc, TblColRef colRef) {
+ PlainParameter single = new PlainParameter();
+ single.type = parameterDesc.type;
+ single.value = parameterDesc.value;
+ single.colRef = colRef;
+ return single;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = type != null ? type.hashCode() : 0;
+ result = 31 * result + (colRef != null ? colRef.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ PlainParameter that = (PlainParameter) o;
+
+ if (type != null ? !type.equals(that.type) : that.type != null)
+ return false;
+
+ if (this.isColumnType()) {
+ if (!that.isColumnType())
+ return false;
+ if (!this.colRef.equals(that.colRef)) {
+ return false;
+ }
+ } else {
+ if (that.isColumnType())
+ return false;
+ if (!this.value.equals(that.value))
+ return false;
+ }
+
+ return true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/kylin-it/src/test/resources/query/sql_distinct/query08.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct/query08.sql b/kylin-it/src/test/resources/query/sql_distinct/query08.sql
new file mode 100644
index 0000000..60f02e7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_distinct/query08.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select cal_dt,
+ sum(price) as GMV,
+ count(1) as TRANS_CNT,
+ count(distinct seller_id, lstg_format_name) as DIST_SELLER_FORMAT
+ from test_kylin_fact
+ group by cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/88a1c71d/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 8d7c597..2c75a14 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -55,6 +56,7 @@ import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.ParamAsMeasureCount;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
@@ -71,6 +73,7 @@ import com.google.common.collect.Sets;
public class OLAPAggregateRel extends Aggregate implements OLAPRel {
private final static Map<String, String> AGGR_FUNC_MAP = new HashMap<String, String>();
+ private final static Map<String, Integer> AGGR_FUNC_PARAM_AS_MEASTURE_MAP = new HashMap<String, Integer>();
static {
AGGR_FUNC_MAP.put("SUM", "SUM");
@@ -84,6 +87,15 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
for (String udaf : udafFactories.keySet()) {
AGGR_FUNC_MAP.put(udaf, udafFactories.get(udaf).getAggrFunctionName());
}
+
+ Map<String, Class<?>> udafs = MeasureTypeFactory.getUDAFs();
+ for (String func : udafs.keySet()) {
+ try {
+ AGGR_FUNC_PARAM_AS_MEASTURE_MAP.put(func, ((ParamAsMeasureCount) (udafs.get(func).newInstance())).getParamAsMeasureCount());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
private static String getSqlFuncName(AggregateCall aggCall) {
@@ -235,12 +247,27 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.aggregations = new ArrayList<FunctionDesc>();
for (AggregateCall aggCall : this.rewriteAggCalls) {
ParameterDesc parameter = null;
+ // By default all args are included, UDFs can define their own in getParamAsMeasureCount method.
if (!aggCall.getArgList().isEmpty()) {
- // TODO: Currently only get the column of first param
- int index = aggCall.getArgList().get(0);
- TblColRef column = inputColumnRowType.getColumnByIndex(index);
- if (!column.isInnerColumn()) {
- parameter = ParameterDesc.newInstance(column);
+ List<TblColRef> columns = Lists.newArrayList();
+ String funcName = getSqlFuncName(aggCall);
+ int columnsCount = aggCall.getArgList().size();
+ if (AGGR_FUNC_PARAM_AS_MEASTURE_MAP.containsKey(funcName)) {
+ int asMeasureCnt = AGGR_FUNC_PARAM_AS_MEASTURE_MAP.get(funcName);
+ if (asMeasureCnt > 0) {
+ columnsCount = asMeasureCnt;
+ } else {
+ columnsCount += asMeasureCnt;
+ }
+ }
+ for (Integer index : aggCall.getArgList().subList(0, columnsCount)) {
+ TblColRef column = inputColumnRowType.getColumnByIndex(index);
+ if (!column.isInnerColumn()) {
+ columns.add(column);
+ }
+ }
+ if (!columns.isEmpty()) {
+ parameter = ParameterDesc.newInstance(columns.toArray(new TblColRef[columns.size()]));
}
}
String expression = getAggrFuncName(aggCall);
@@ -341,10 +368,11 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AggregateCall aggCall = this.rewriteAggCalls.get(i);
if (!aggCall.getArgList().isEmpty()) {
- int index = aggCall.getArgList().get(0);
- TblColRef column = inputColumnRowType.getColumnByIndex(index);
- if (!column.isInnerColumn()) {
- this.context.metricsColumns.add(column);
+ for (Integer index : aggCall.getArgList()) {
+ TblColRef column = inputColumnRowType.getColumnByIndex(index);
+ if (!column.isInnerColumn()) {
+ this.context.metricsColumns.add(column);
+ }
}
}
}
@@ -385,18 +413,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
return aggCall;
}
- // rebuild parameters
- List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
- if (func.needRewriteField()) {
- RelDataTypeField field = getInput().getRowType().getField(func.getRewriteFieldName(), true, false);
- if (newArgList.isEmpty()) {
- newArgList.add(field.getIndex());
- } else {
- // only the first column got overwritten
- newArgList.set(0, field.getIndex());
- }
- }
-
// rebuild function
String callName = getSqlFuncName(aggCall);
RelDataType fieldType = aggCall.getType();
@@ -408,12 +424,40 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
newAgg = createCustomAggFunction(callName, fieldType, udafMap.get(callName));
}
+ // rebuild parameters
+ List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
+ if (udafMap != null && udafMap.containsKey(callName)) {
+ newArgList = truncArgList(newArgList, udafMap.get(callName));
+ }
+ if (func.needRewriteField()) {
+ RelDataTypeField field = getInput().getRowType().getField(func.getRewriteFieldName(), true, false);
+ if (newArgList.isEmpty()) {
+ newArgList.add(field.getIndex());
+ } else {
+ // TODO: only the first column got overwritten
+ newArgList.set(0, field.getIndex());
+ }
+ }
+
// rebuild aggregate call
@SuppressWarnings("deprecation")
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, callName);
return newAggCall;
}
+ /**
+ * truncate Arg List according to UDAF's "add" method parameter count
+ */
+ private List<Integer> truncArgList(List<Integer> argList, Class<?> udafClazz) {
+ int argListLength = argList.size();
+ for (Method method : udafClazz.getMethods()) {
+ if (method.getName().equals("add")) {
+ argListLength = Math.min(method.getParameterTypes().length - 1, argListLength);
+ }
+ }
+ return argList.subList(0, argListLength);
+ }
+
private SqlAggFunction createCustomAggFunction(String funcName, RelDataType returnType, Class<?> customAggFuncClz) {
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
SqlIdentifier sqlIdentifier = new SqlIdentifier(funcName, new SqlParserPos(1, 1));
[11/35] kylin git commit: KYLIN-2525 tolerate job metadata exception
in StorageCleanupJob
Posted by li...@apache.org.
KYLIN-2525 tolerate job metadata exception in StorageCleanupJob
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fa3ee3ff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fa3ee3ff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fa3ee3ff
Branch: refs/heads/master-hbase0.98
Commit: fa3ee3ffbbb11ac3d4dc79dbb719f56a7e913857
Parents: c6b4592
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 18:03:59 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Mar 29 21:02:23 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/tool/StorageCleanupJob.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fa3ee3ff/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 477c58a..f1a3ebe 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -46,6 +46,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -57,7 +58,6 @@ import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.source.hive.HiveClientFactory;
-import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.source.hive.IHiveClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -276,9 +276,14 @@ public class StorageCleanupJob extends AbstractApplication {
sb.append(jobId).append("(").append(state).append("), ");
}
- String segmentId = getSegmentIdFromJobId(jobId);
- if (segmentId != null) {//some jobs are not cubing jobs
- segmentId2JobId.put(segmentId, jobId);
+ try {
+ String segmentId = getSegmentIdFromJobId(jobId);
+ if (segmentId != null) {//some jobs are not cubing jobs
+ segmentId2JobId.put(segmentId, jobId);
+ }
+ } catch (Exception ex) {
+ logger.warn("Failed to find segment ID from job ID " + jobId + ", ignore it");
+ // some older version job metadata may fail to read, ignore it
}
}
logger.info("Working jobIDs: " + workingJobList);
[04/35] kylin git commit: Minor,
use util class to hold static job function
Posted by li...@apache.org.
Minor, use util class to hold static job function
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e03c718
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e03c718
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e03c718
Branch: refs/heads/master-hbase0.98
Commit: 9e03c718f86725e7b5d0b96950d8372b7bb77428
Parents: b1cc0dd
Author: Yifan Zhang <ev...@gmail.com>
Authored: Mon Mar 27 10:50:12 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Mon Mar 27 15:50:53 2017 +0800
----------------------------------------------------------------------
.gitignore | 4 +-
.../apache/kylin/rest/service/JobService.java | 103 +--------------
.../kylin/rest/util/JobInfoConverter.java | 130 +++++++++++++++++++
3 files changed, 136 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e03c718/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index bb8b3b0..3679a13 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,7 +14,7 @@ release.properties
#IDEA
*.iml
-#.settings
+.settings
# External tool builders
.externalToolBuilders/
@@ -88,4 +88,4 @@ build/commit_SHA1
dist/
tomcat/
webapp/app/kylin-servlet.xml
-webapp/app/web.xml
\ No newline at end of file
+webapp/app/web.xml
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e03c718/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 9836766..1e91b43 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -37,15 +37,11 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.Scheduler;
import org.apache.kylin.job.SchedulerFactory;
-import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.exception.JobException;
@@ -59,6 +55,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.util.JobInfoConverter;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
@@ -275,108 +272,16 @@ public class JobService extends BasicService implements InitializingBean {
result.setSubmitter(cubeJob.getSubmitter());
result.setUuid(cubeJob.getId());
result.setType(CubeBuildTypeEnum.BUILD);
- result.setStatus(parseToJobStatus(job.getStatus()));
+ result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
result.setDuration(cubeJob.getDuration() / 1000);
for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
AbstractExecutable task = cubeJob.getTasks().get(i);
- result.addStep(parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
+ result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
}
return result;
}
- private JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
- if (job == null) {
- return null;
- }
- Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
- CubingJob cubeJob = (CubingJob) job;
- Output output = outputs.get(job.getId());
- final JobInstance result = new JobInstance();
- result.setName(job.getName());
- result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
- result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
- result.setLastModified(output.getLastModified());
- result.setSubmitter(cubeJob.getSubmitter());
- result.setUuid(cubeJob.getId());
- result.setType(CubeBuildTypeEnum.BUILD);
- result.setStatus(parseToJobStatus(output.getState()));
- result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- result.setExecStartTime(AbstractExecutable.getStartTime(output));
- result.setExecEndTime(AbstractExecutable.getEndTime(output));
- result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
- for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
- AbstractExecutable task = cubeJob.getTasks().get(i);
- result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
- }
- return result;
- }
-
- private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
- Preconditions.checkNotNull(stepOutput);
- JobInstance.JobStep result = new JobInstance.JobStep();
- result.setId(task.getId());
- result.setName(task.getName());
- result.setSequenceID(i);
- result.setStatus(parseToJobStepStatus(stepOutput.getState()));
- for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
- if (entry.getKey() != null && entry.getValue() != null) {
- result.putInfo(entry.getKey(), entry.getValue());
- }
- }
- result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
- result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
- if (task instanceof ShellExecutable) {
- result.setExecCmd(((ShellExecutable) task).getCmd());
- }
- if (task instanceof MapReduceExecutable) {
- result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
- result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- }
- if (task instanceof HadoopShellExecutable) {
- result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
- }
- return result;
- }
-
- private JobStatusEnum parseToJobStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- return JobStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-
- private JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- return JobStepStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
public void resumeJob(JobInstance job) throws IOException, JobException {
getExecutableManager().resumeJob(job.getId());
@@ -448,7 +353,7 @@ public class JobService extends BasicService implements InitializingBean {
return Lists.newArrayList(FluentIterable.from(searchCubingJobs(cubeNameSubstring, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
- return parseToJobInstance(cubingJob, allOutputs);
+ return JobInfoConverter.parseToJobInstance(cubingJob, allOutputs);
}
}));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e03c718/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java b/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
new file mode 100644
index 0000000..e13c9c3
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.util;
+
+import java.util.Map;
+
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
+import org.apache.kylin.engine.mr.common.MapReduceExecutable;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.JobStatusEnum;
+import org.apache.kylin.job.constant.JobStepStatusEnum;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.Output;
+
+import com.google.common.base.Preconditions;
+
+public class JobInfoConverter {
+ public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
+ if (job == null) {
+ return null;
+ }
+ Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
+ CubingJob cubeJob = (CubingJob) job;
+ Output output = outputs.get(job.getId());
+ final JobInstance result = new JobInstance();
+ result.setName(job.getName());
+ result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
+ result.setLastModified(output.getLastModified());
+ result.setSubmitter(cubeJob.getSubmitter());
+ result.setUuid(cubeJob.getId());
+ result.setType(CubeBuildTypeEnum.BUILD);
+ result.setStatus(parseToJobStatus(output.getState()));
+ result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setExecStartTime(AbstractExecutable.getStartTime(output));
+ result.setExecEndTime(AbstractExecutable.getEndTime(output));
+ result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
+ for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
+ AbstractExecutable task = cubeJob.getTasks().get(i);
+ result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
+ }
+ return result;
+ }
+
+ public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
+ Preconditions.checkNotNull(stepOutput);
+ JobInstance.JobStep result = new JobInstance.JobStep();
+ result.setId(task.getId());
+ result.setName(task.getName());
+ result.setSequenceID(i);
+ result.setStatus(parseToJobStepStatus(stepOutput.getState()));
+ for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
+ if (entry.getKey() != null && entry.getValue() != null) {
+ result.putInfo(entry.getKey(), entry.getValue());
+ }
+ }
+ result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
+ result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
+ if (task instanceof ShellExecutable) {
+ result.setExecCmd(((ShellExecutable) task).getCmd());
+ }
+ if (task instanceof MapReduceExecutable) {
+ result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
+ result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ }
+ if (task instanceof HadoopShellExecutable) {
+ result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
+ }
+ return result;
+ }
+
+ public static JobStatusEnum parseToJobStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+
+ public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStepStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+}
[14/35] kylin git commit: KYLIN-2527 Speedup LookupStringTable,
use HashMap instead of ConcurrentHashMap
Posted by li...@apache.org.
KYLIN-2527 Speedup LookupStringTable, use HashMap instead of ConcurrentHashMap
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85a341b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85a341b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85a341b9
Branch: refs/heads/master-hbase0.98
Commit: 85a341b9513f36857e1ba06ab0000cc2d4dfe400
Parents: 9147d3e
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 31 16:45:02 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 16:45:02 2017 +0800
----------------------------------------------------------------------
.../storage/gtrecord/CubeTupleConverter.java | 140 ++++++++++++-------
1 file changed, 89 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85a341b9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 280718f..526a554 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -18,6 +18,8 @@
package org.apache.kylin.storage.gtrecord;
+import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -25,28 +27,34 @@ import java.util.Set;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dict.lookup.SnapshotManager;
+import org.apache.kylin.dict.lookup.SnapshotTable;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.source.ReadableTable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-/**
- * convert GTRecord to tuple
- */
public class CubeTupleConverter implements ITupleConverter {
+ private static final Logger logger = LoggerFactory.getLogger(CubeTupleConverter.class);
+
final CubeSegment cubeSeg;
final Cuboid cuboid;
final TupleInfo tupleInfo;
@@ -63,7 +71,7 @@ public class CubeTupleConverter implements ITupleConverter {
private final int nSelectedDims;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
+ Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
this.tupleInfo = returnTupleInfo;
@@ -92,11 +100,6 @@ public class CubeTupleConverter implements ITupleConverter {
int dimIndex = mapping.getIndexOf(dim);
gtColIdx[i] = dimIndex;
tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
-
- // if (tupleIdx[iii] == -1) {
- // throw new IllegalStateException("dim not used in tuple:" + dim);
- // }
-
i++;
}
@@ -209,55 +212,54 @@ public class CubeTupleConverter implements ITupleConverter {
return null;
switch (deriveInfo.type) {
- case LOOKUP:
- return new IDerivedColumnFiller() {
- CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
- int[] derivedColIdx = initDerivedColIdx();
- Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
-
- private int[] initDerivedColIdx() {
- int[] idx = new int[deriveInfo.columns.length];
- for (int i = 0; i < idx.length; i++) {
- idx[i] = deriveInfo.columns[i].getColumnDesc().getZeroBasedIndex();
- }
- return idx;
+ case LOOKUP:
+ return new IDerivedColumnFiller() {
+ LookupStringTable lookupTable = getLookupTable(cubeSeg, deriveInfo.join);
+ int[] derivedColIdx = initDerivedColIdx();
+ Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
+
+ private int[] initDerivedColIdx() {
+ int[] idx = new int[deriveInfo.columns.length];
+ for (int i = 0; i < idx.length; i++) {
+ idx[i] = deriveInfo.columns[i].getColumnDesc().getZeroBasedIndex();
}
+ return idx;
+ }
- @Override
- public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
- for (int i = 0; i < hostTmpIdx.length; i++) {
- lookupKey.data[i] = CubeTupleConverter.toString(gtValues[hostTmpIdx[i]]);
- }
+ @Override
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
+ for (int i = 0; i < hostTmpIdx.length; i++) {
+ lookupKey.data[i] = CubeTupleConverter.toString(gtValues[hostTmpIdx[i]]);
+ }
- String[] lookupRow = lookupTable.getRow(lookupKey);
+ String[] lookupRow = lookupTable.getRow(lookupKey);
- if (lookupRow != null) {
- for (int i = 0; i < derivedTupleIdx.length; i++) {
- if (derivedTupleIdx[i] >= 0) {
- String value = lookupRow[derivedColIdx[i]];
- tuple.setDimensionValue(derivedTupleIdx[i], value);
- }
+ if (lookupRow != null) {
+ for (int i = 0; i < derivedTupleIdx.length; i++) {
+ if (derivedTupleIdx[i] >= 0) {
+ String value = lookupRow[derivedColIdx[i]];
+ tuple.setDimensionValue(derivedTupleIdx[i], value);
}
- } else {
- for (int i = 0; i < derivedTupleIdx.length; i++) {
- if (derivedTupleIdx[i] >= 0) {
- tuple.setDimensionValue(derivedTupleIdx[i], null);
- }
+ }
+ } else {
+ for (int i = 0; i < derivedTupleIdx.length; i++) {
+ if (derivedTupleIdx[i] >= 0) {
+ tuple.setDimensionValue(derivedTupleIdx[i], null);
}
}
}
- };
- case PK_FK:
- return new IDerivedColumnFiller() {
- @Override
- public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
- // composite keys are split, so only copy [0] is enough, see CubeDesc.initDimensionColumns()
- tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(gtValues[hostTmpIdx[0]]));
- }
- };
- default:
- throw new IllegalArgumentException();
+ }
+ };
+ case PK_FK:
+ return new IDerivedColumnFiller() {
+ @Override
+ public void fillDerivedColumns(Object[] gtValues, Tuple tuple) {
+ // composite keys are split, so only copy [0] is enough, see CubeDesc.initDimensionColumns()
+ tuple.setDimensionValue(derivedTupleIdx[0], CubeTupleConverter.toString(gtValues[hostTmpIdx[0]]));
+ }
+ };
+ default:
+ throw new IllegalArgumentException();
}
}
@@ -271,6 +273,42 @@ public class CubeTupleConverter implements ITupleConverter {
return -1;
}
+ public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
+ long ts = System.currentTimeMillis();
+
+ MetadataManager metaMgr = MetadataManager.getInstance(cubeSeg.getCubeInstance().getConfig());
+ SnapshotManager snapshotMgr = SnapshotManager.getInstance(cubeSeg.getCubeInstance().getConfig());
+
+ String tableName = join.getPKSide().getTableIdentity();
+ String[] pkCols = join.getPrimaryKey();
+ String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
+ if (snapshotResPath == null)
+ throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
+
+ try {
+ SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotResPath);
+ TableDesc tableDesc = metaMgr.getTableDesc(tableName);
+ EnhancedStringLookupTable enhancedStringLookupTable = new EnhancedStringLookupTable(tableDesc, pkCols, snapshot);
+ logger.info("Time to get lookup up table for {} is {} ", join.getPKSide().getTableName(), (System.currentTimeMillis() - ts));
+ return enhancedStringLookupTable;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
+ }
+ }
+
+ private static class EnhancedStringLookupTable extends LookupStringTable {
+
+ public EnhancedStringLookupTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
+ super(tableDesc, keyColumns, table);
+ }
+
+ @Override
+ protected void init() throws IOException {
+ this.data = new HashMap<>();
+ super.init();
+ }
+ }
+
private static String toString(Object o) {
return o == null ? null : o.toString();
}
[27/35] kylin git commit: minor,
allow DebugTomcat on sandbox in different LAN
Posted by li...@apache.org.
minor, allow DebugTomcat on sandbox in different LAN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/551f5ed2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/551f5ed2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/551f5ed2
Branch: refs/heads/master-hbase0.98
Commit: 551f5ed2e7922f63e260b5adf1e04c8a86a77298
Parents: 7af5529
Author: lidongsjtu <li...@apache.org>
Authored: Sat Apr 1 20:55:58 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Apr 4 10:55:37 2017 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hdfs-site.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/551f5ed2/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1d9040a..c06222e 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -15,6 +15,10 @@
limitations under the License.
-->
<configuration>
+ <property>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>true</value>
+ </property>
<property>
<name>dfs.block.access.token.enable</name>
[19/35] kylin git commit: minor,
code refactor in GTCubeStorageQueryBase
Posted by li...@apache.org.
minor, code refactor in GTCubeStorageQueryBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4cce762
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4cce762
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4cce762
Branch: refs/heads/master-hbase0.98
Commit: d4cce762addda1c819939e767220864dad551503
Parents: f6ee76c
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 30 17:25:22 2017 +0800
Committer: hongbin ma <ma...@kyligence.io>
Committed: Fri Mar 31 18:34:37 2017 +0800
----------------------------------------------------------------------
.../gtrecord/GTCubeStorageQueryBase.java | 68 +++++++-------
.../gtrecord/GTCubeStorageQueryRequest.java | 94 ++++++++++++++++++++
2 files changed, 130 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ecf1ad3..1b4964f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -72,6 +72,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);
+
+ List<CubeSegmentScanner> scanners = Lists.newArrayList();
+ for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
+ CubeSegmentScanner scanner;
+
+ if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
+ continue;
+ }
+
+ scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getContext());
+ scanners.add(scanner);
+ }
+
+ if (scanners.isEmpty())
+ return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+
+ return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getMetrics(), returnTupleInfo, request.getContext());
+ }
+
+ protected abstract String getGTStorage();
+
+ protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
context.setStorageQuery(this);
//deal with participant columns in subquery join
@@ -106,7 +130,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
dimensionsD.addAll(groupsD);
dimensionsD.addAll(otherDimsD);
Cuboid cuboid = Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
- logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, otherDimsD={}", cubeInstance.getName(), cuboid.getId(), groupsD, otherDimsD);
context.setCuboid(cuboid);
// set whether to aggr at storage
@@ -115,35 +138,21 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
Set<TblColRef> loosenedColumnD = Sets.newHashSet();
+ Set<TblColRef> filterColumnD = Sets.newHashSet();
TupleFilter filterD = translateDerived(filter, loosenedColumnD);
groupsD.addAll(loosenedColumnD);
+ TupleFilter.collectColumns(filter, filterColumnD);
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
// set query deadline
context.setDeadline(cubeInstance);
- List<CubeSegmentScanner> scanners = Lists.newArrayList();
- for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
- CubeSegmentScanner scanner;
-
- if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
- logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
- continue;
- }
-
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context);
- scanners.add(scanner);
- }
+ logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
- if (scanners.isEmpty())
- return ITupleIterator.EMPTY_TUPLE_ITERATOR;
-
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
}
- protected abstract String getGTStorage();
-
protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
}
@@ -235,19 +244,14 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) {
-
- logger.info("GroupD :" + groupD);
- logger.info("SingleValueD :" + singleValueD);
- logger.info("Cuboid columns :" + cuboid.getColumns());
-
HashSet<TblColRef> temp = Sets.newHashSet();
temp.addAll(groupD);
temp.addAll(singleValueD);
if (cuboid.getColumns().size() == temp.size()) {
- logger.info("Does not need storage aggregation");
+ logger.debug("Does not need storage aggregation");
return false;
} else {
- logger.info("Need storage aggregation");
+ logger.debug("Need storage aggregation");
return true;
}
}
@@ -326,30 +330,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
if (!TupleFilter.isEvaluableRecursively(filter)) {
possible = false;
- logger.info("Storage limit push down is impossible because the filter isn't evaluable");
+ logger.debug("Storage limit push down is impossible because the filter isn't evaluable");
}
if (!loosenedColumnD.isEmpty()) { // KYLIN-2173
possible = false;
- logger.info("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD);
+ logger.debug("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD);
}
if (context.hasSort()) {
possible = false;
- logger.info("Storage limit push down is impossible because the query has order by");
+ logger.debug("Storage limit push down is impossible because the query has order by");
}
// derived aggregation is bad, unless expanded columns are already in group by
if (!groups.containsAll(derivedPostAggregation)) {
possible = false;
- logger.info("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation);
+ logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation);
}
//if groupsD is clustered at "head" of the rowkey, then limit push down is possible
int size = groupsD.size();
if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) {
possible = false;
- logger.info("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD //
+ logger.debug("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD //
+ " with cuboid columns: " + cuboid.getColumns());
}
@@ -357,7 +361,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
for (FunctionDesc functionDesc : functionDescs) {
if (functionDesc.isDimensionAsMetric()) {
possible = false;
- logger.info("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc);
+ logger.debug("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
new file mode 100644
index 0000000..c4d81d2
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.Serializable;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
+
+public class GTCubeStorageQueryRequest implements Serializable {
+ private Cuboid cuboid;
+ private Set<TblColRef> dimensions;
+ private Set<TblColRef> groups;
+ private Set<FunctionDesc> metrics;
+ private TupleFilter filter;
+ private StorageContext context;
+
+ public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, TupleFilter filter, StorageContext context) {
+ this.cuboid = cuboid;
+ this.dimensions = dimensions;
+ this.groups = groups;
+ this.metrics = metrics;
+ this.filter = filter;
+ this.context = context;
+ }
+
+ public Cuboid getCuboid() {
+ return cuboid;
+ }
+
+ public void setCuboid(Cuboid cuboid) {
+ this.cuboid = cuboid;
+ }
+
+ public Set<TblColRef> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(Set<TblColRef> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public Set<TblColRef> getGroups() {
+ return groups;
+ }
+
+ public void setGroups(Set<TblColRef> groups) {
+ this.groups = groups;
+ }
+
+ public Set<FunctionDesc> getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(Set<FunctionDesc> metrics) {
+ this.metrics = metrics;
+ }
+
+ public TupleFilter getFilter() {
+ return filter;
+ }
+
+ public void setFilter(TupleFilter filter) {
+ this.filter = filter;
+ }
+
+ public StorageContext getContext() {
+ return context;
+ }
+
+ public void setContext(StorageContext context) {
+ this.context = context;
+ }
+}
[32/35] kylin git commit: KYLIN-2165 fix UT
Posted by li...@apache.org.
KYLIN-2165 fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93cc5ab6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93cc5ab6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93cc5ab6
Branch: refs/heads/master-hbase0.98
Commit: 93cc5ab6a454ecf99a492fa1eb4161a13bf91247
Parents: 7129ea6
Author: gaodayue <ga...@meituan.com>
Authored: Fri Apr 7 12:11:34 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Fri Apr 7 12:11:34 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/93cc5ab6/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 0a03461..2f966ab 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -138,7 +138,7 @@ public class HiveMRInput implements IMRInput {
@Override
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
- final KylinConfig kylinConfig = CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
+ final KylinConfig kylinConfig = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig();
this.conf = new JobEngineConfig(kylinConfig);
// create flat table first, then count and redistribute
[17/35] kylin git commit: KYLIN-2521 upgrade calcite to 1.12
Posted by li...@apache.org.
KYLIN-2521 upgrade calcite to 1.12
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/19d5b3de
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/19d5b3de
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/19d5b3de
Branch: refs/heads/master-hbase0.98
Commit: 19d5b3dee38a40c0a344b5f340c4806514290cc3
Parents: 85a341b
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 31 16:51:03 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 16:51:03 2017 +0800
----------------------------------------------------------------------
.../calcite/sql2rel/SqlToRelConverter.java | 2140 +++++++++---------
pom.xml | 2 +-
.../optrule/AggregateMultipleExpandRule.java | 2 +-
.../kylin/query/relnode/OLAPTableScan.java | 3 +-
.../relnode/OLAPToEnumerableConverter.java | 6 +
5 files changed, 1133 insertions(+), 1020 deletions(-)
----------------------------------------------------------------------
[21/35] kylin git commit: Merge commit
'782a97482ca1e92cf24e04badd8ee48c9b829f46'
Posted by li...@apache.org.
Merge commit '782a97482ca1e92cf24e04badd8ee48c9b829f46'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ee5397bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ee5397bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ee5397bc
Branch: refs/heads/master-hbase0.98
Commit: ee5397bc6207c9cdf57cdce93bcda6bcac58a5a9
Parents: d4cce76 782a974
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 31 20:00:16 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 20:00:59 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../kylin/common/util/ImmutableBitSet.java | 29 +-
.../org/apache/kylin/GTForwardingScanner.java | 56 ++++
.../java/org/apache/kylin/cube/CubeManager.java | 8 +-
.../kylin/cube/gridtable/CubeGridTable.java | 18 --
.../gridtable/CuboidToGridTableMapping.java | 18 ++
.../cube/inmemcubing/InMemCubeBuilder.java | 6 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 11 +-
.../kylin/gridtable/GTAggregateScanner.java | 16 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 22 +-
.../org/apache/kylin/gridtable/GTRecord.java | 80 ++---
.../apache/kylin/gridtable/GTScanRequest.java | 13 +
.../gridtable/GTStreamAggregateScanner.java | 219 ++++++++++++++
.../kylin/gridtable/GTScanReqSerDerTest.java | 4 +-
.../apache/kylin/measure/hllc/HLLCounter.java | 54 ++--
.../apache/kylin/storage/StorageContext.java | 8 +
.../storage/gtrecord/CubeScanRangePlanner.java | 3 +-
.../storage/gtrecord/CubeSegmentScanner.java | 7 +-
.../storage/gtrecord/CubeTupleConverter.java | 26 +-
.../gtrecord/GTCubeStorageQueryBase.java | 35 ++-
.../kylin/storage/gtrecord/ITupleConverter.java | 3 +-
.../gtrecord/PartitionResultIterator.java | 59 ++++
.../kylin/storage/gtrecord/ScannerWorker.java | 5 +-
.../gtrecord/SegmentCubeTupleIterator.java | 71 ++++-
.../SortMergedPartitionResultIterator.java | 81 +++++
.../gtrecord/StorageResponseGTScatter.java | 83 ++---
.../mr/steps/FactDistinctColumnsMapper.java | 31 +-
.../mr/steps/NewCubeSamplingMethodTest.java | 299 +++++++++++++++++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 5 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 5 +-
.../apache/kylin/tool/StorageCleanupJob.java | 13 +-
33 files changed, 1047 insertions(+), 254 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/build/conf/kylin.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --cc core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f4480c8,3397adc..3e62ea7
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@@ -118,20 -114,14 +114,24 @@@ public class GTRecord implements Compar
return result;
}
++
+ /** decode and return the values of this record */
+ public Object[] getValues(int[] selectedColumns, Object[] result) {
+ assert selectedColumns.length <= result.length;
+ for (int i = 0; i < selectedColumns.length; i++) {
- int c = selectedColumns[i];
- if (cols[c].array() == null) {
- result[i] = null;
- } else {
- result[i] = info.codeSystem.decodeColumnValue(c, cols[c].asBuffer());
- }
++ result[i] = decodeValue(selectedColumns[i]);
+ }
+ return result;
+ }
+
+ public Object decodeValue(int c) {
+ ByteArray col = cols[c];
+ if (col != null && col.array() != null) {
+ return info.codeSystem.decodeColumnValue(c, col.asBuffer());
+ }
+ return null;
+ }
+
public int sizeOf(ImmutableBitSet selectedCols) {
int size = 0;
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --cc core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 526a554,b762e5c..7df80ca
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@@ -27,20 -25,14 +27,18 @@@ import java.util.Set
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
- import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.dict.lookup.SnapshotManager;
+import org.apache.kylin.dict.lookup.SnapshotTable;
- import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
@@@ -51,10 -40,11 +49,13 @@@ import org.slf4j.LoggerFactory
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+ /**
+ * Convert Object[] (decoded GTRecord) to tuple
+ */
public class CubeTupleConverter implements ITupleConverter {
+ private static final Logger logger = LoggerFactory.getLogger(CubeTupleConverter.class);
+
final CubeSegment cubeSeg;
final Cuboid cuboid;
final TupleInfo tupleInfo;
@@@ -71,9 -60,11 +71,10 @@@
private final int nSelectedDims;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx,
- TupleInfo returnTupleInfo) {
++ Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
+ this.gtColIdx = gtColIdx;
this.tupleInfo = returnTupleInfo;
this.derivedColFillers = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ee5397bc/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --cc core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 1b4964f,d91a0b4..065fd6e
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@@ -72,30 -72,6 +72,28 @@@ public abstract class GTCubeStorageQuer
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);
+
+ List<CubeSegmentScanner> scanners = Lists.newArrayList();
+ for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
+ CubeSegmentScanner scanner;
+
+ if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
+ continue;
+ }
+
+ scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getContext());
+ scanners.add(scanner);
+ }
+
+ if (scanners.isEmpty())
+ return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+
+ return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getMetrics(), returnTupleInfo, request.getContext());
+ }
+
- protected abstract String getGTStorage();
-
+ protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
context.setStorageQuery(this);
//deal with participant columns in subquery join
@@@ -148,13 -125,29 +148,15 @@@
// set query deadline
context.setDeadline(cubeInstance);
- List<CubeSegmentScanner> scanners = Lists.newArrayList();
- for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
- CubeSegmentScanner scanner;
-
- if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
- logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
- continue;
- }
+ logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context);
- scanners.add(scanner);
- }
-
- if (scanners.isEmpty())
- return ITupleIterator.EMPTY_TUPLE_ITERATOR;
-
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
}
- protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
- return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+ protected abstract String getGTStorage();
+
+ protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) {
+ return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo);
}
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
[10/35] kylin git commit: KYLIN-2525 skip signature check for older
version of cubes
Posted by li...@apache.org.
KYLIN-2525 skip signature check for older version of cubes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6b4592f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6b4592f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6b4592f
Branch: refs/heads/master-hbase0.98
Commit: c6b4592fca047d1fc4094a0812774ddd2d8917f9
Parents: a776ef6
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 11:57:10 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Mar 29 21:02:16 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 8 ++++----
.../main/java/org/apache/kylin/cube/model/CubeDesc.java | 11 +++++++++--
2 files changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6b4592f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 073f516..0a94fb2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -761,7 +761,7 @@ public class CubeManager implements IRealizationProvider {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(cube.getDescName());
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", cube.getDescName(), cubeName);
- if (!isITTestCube(cubeName))
+ if (!isSpecialTestCube(cubeName))
checkState(cubeDesc.getName().equals(cubeName), "cube name '%s' must be same as descriptor name '%s', but it is not", cubeName, cubeDesc.getName());
if (!cubeDesc.getError().isEmpty()) {
@@ -791,9 +791,9 @@ public class CubeManager implements IRealizationProvider {
}
}
- private boolean isITTestCube(String cubeName) {
- return config.isDevEnv() //
- && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
+ private boolean isSpecialTestCube(String cubeName) {
+ return cubeName.equals("kylin_sales_cube") //
+ || config.isDevEnv() && (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
}
private MetadataManager getMetadataManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6b4592f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index c1469fe..b391055 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -479,8 +479,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return true;
}
- if (KylinVersion.getCurrentVersion().isCompatibleWith(new KylinVersion(getVersion())) && !KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new KylinVersion(getVersion()))) {
- logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion());
+ KylinVersion cubeVersion = new KylinVersion(getVersion());
+ KylinVersion kylinVersion = KylinVersion.getCurrentVersion();
+ if (!kylinVersion.isCompatibleWith(cubeVersion)) {
+ logger.info("checkSignature on {} is skipped as the its version {} is different from kylin version {}", getName(), cubeVersion, kylinVersion);
+ return true;
+ }
+
+ if (kylinVersion.isCompatibleWith(cubeVersion) && !kylinVersion.isSignatureCompatibleWith(cubeVersion)) {
+ logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), cubeVersion);
return true;
}
[20/35] kylin git commit: KYLIN-2529 Allow thread-local override of
KylinConfig
Posted by li...@apache.org.
KYLIN-2529 Allow thread-local override of KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e8b6a5e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e8b6a5e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e8b6a5e
Branch: refs/heads/master-hbase0.98
Commit: 1e8b6a5e7b1350930c68e750fb1ce75c27428082
Parents: 782a974
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 30 19:54:34 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 31 19:03:54 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 37 ++++++++++++++------
.../apache/kylin/common/KylinConfigTest.java | 31 +++++++++++++++-
2 files changed, 56 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b6a5e/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index c6b1511..a9a0c45 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -50,30 +50,41 @@ public class KylinConfig extends KylinConfigBase {
public static final String KYLIN_CONF = "KYLIN_CONF";
// static cached instances
- private static KylinConfig ENV_INSTANCE = null;
+ private static KylinConfig SYS_ENV_INSTANCE = null;
+
+ // thread-local instances, will override SYS_ENV_INSTANCE
+ private static final transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
- if (ENV_INSTANCE == null) {
+ KylinConfig config = THREAD_ENV_INSTANCE.get();
+ if (config != null) {
+ return config;
+ }
+
+ if (SYS_ENV_INSTANCE == null) {
try {
- KylinConfig config = new KylinConfig();
+ config = new KylinConfig();
config.reloadKylinConfig(getKylinProperties());
logger.info("Initialized a new KylinConfig from getInstanceFromEnv : " + System.identityHashCode(config));
- ENV_INSTANCE = config;
+ SYS_ENV_INSTANCE = config;
} catch (IllegalArgumentException e) {
throw new IllegalStateException("Failed to find KylinConfig ", e);
}
}
- return ENV_INSTANCE;
+ return SYS_ENV_INSTANCE;
}
}
//Only used in test cases!!!
public static void destroyInstance() {
- logger.info("Destory KylinConfig");
- dumpStackTrace();
- ENV_INSTANCE = null;
+ synchronized (KylinConfig.class) {
+ logger.info("Destroy KylinConfig");
+ dumpStackTrace();
+ SYS_ENV_INSTANCE = null;
+ THREAD_ENV_INSTANCE.remove();
+ }
}
public enum UriType {
@@ -158,12 +169,12 @@ public class KylinConfig extends KylinConfigBase {
public static void setKylinConfigInEnvIfMissing(Properties prop) {
synchronized (KylinConfig.class) {
- if (ENV_INSTANCE == null) {
+ if (SYS_ENV_INSTANCE == null) {
try {
KylinConfig config = new KylinConfig();
config.reloadKylinConfig(prop);
- logger.info("Resetting ENV_INSTANCE by a input stream: " + System.identityHashCode(config));
- ENV_INSTANCE = config;
+ logger.info("Resetting SYS_ENV_INSTANCE by a input stream: " + System.identityHashCode(config));
+ SYS_ENV_INSTANCE = config;
} catch (IllegalArgumentException e) {
throw new IllegalStateException("Failed to find KylinConfig ", e);
}
@@ -177,6 +188,10 @@ public class KylinConfig extends KylinConfigBase {
setKylinConfigInEnvIfMissing(props);
}
+ public static void setKylinConfigThreadLocal(KylinConfig config) {
+ THREAD_ENV_INSTANCE.set(config);
+ }
+
public static KylinConfig createKylinConfig(String propsInStr) throws IOException {
Properties props = new Properties();
props.load(new StringReader(propsInStr));
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e8b6a5e/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 7e4b444..3976c6c 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.util.Map;
+import java.util.Properties;
import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase;
import org.junit.Test;
@@ -86,7 +87,7 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
public void testGetMetadataUrlPrefix() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
final String default_metadata_prefix = "kylin_metadata";
-
+
config.setMetadataUrl("testMetaPrefix@hbase");
assertEquals("testMetaPrefix", config.getMetadataUrlPrefix());
@@ -96,4 +97,32 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
config.setMetadataUrl("/kylin/temp");
assertEquals(default_metadata_prefix, config.getMetadataUrlPrefix());
}
+
+ @Test
+ public void testThreadLocalOverride() {
+ final String metadata1 = "meta1";
+ final String metadata2 = "meta2";
+
+ // set system KylinConfig
+ KylinConfig sysConfig = KylinConfig.getInstanceFromEnv();
+ sysConfig.setMetadataUrl(metadata1);
+
+ assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl());
+
+ // test thread-local override
+ KylinConfig threadConfig = KylinConfig.createKylinConfig(new Properties());
+ threadConfig.setMetadataUrl(metadata2);
+ KylinConfig.setKylinConfigThreadLocal(threadConfig);
+
+ assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl());
+
+ // other threads still use system KylinConfig
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ System.out.println("Started new thread.");
+ assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl());
+ }
+ }).start();
+ }
}
[06/35] kylin git commit: minor,
move JobInfoConverter to kylin-engine-mr
Posted by li...@apache.org.
minor, move JobInfoConverter to kylin-engine-mr
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/598b32ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/598b32ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/598b32ef
Branch: refs/heads/master-hbase0.98
Commit: 598b32efa21ebd87e76ac5798b30f472c6e29de2
Parents: 997d429
Author: Yifan Zhang <ev...@gmail.com>
Authored: Wed Mar 29 10:29:16 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Wed Mar 29 10:47:10 2017 +0800
----------------------------------------------------------------------
.../engine/mr/common/JobInfoConverter.java | 128 ++++++++++++++++++
.../apache/kylin/rest/service/JobService.java | 2 +-
.../kylin/rest/util/JobInfoConverter.java | 130 -------------------
3 files changed, 129 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
new file mode 100644
index 0000000..189e019
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.util.Map;
+
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.JobStatusEnum;
+import org.apache.kylin.job.constant.JobStepStatusEnum;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.job.execution.Output;
+
+import com.google.common.base.Preconditions;
+
+public class JobInfoConverter {
+ public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
+ if (job == null) {
+ return null;
+ }
+ Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
+ CubingJob cubeJob = (CubingJob) job;
+ Output output = outputs.get(job.getId());
+ final JobInstance result = new JobInstance();
+ result.setName(job.getName());
+ result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
+ result.setLastModified(output.getLastModified());
+ result.setSubmitter(cubeJob.getSubmitter());
+ result.setUuid(cubeJob.getId());
+ result.setType(CubeBuildTypeEnum.BUILD);
+ result.setStatus(parseToJobStatus(output.getState()));
+ result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ result.setExecStartTime(AbstractExecutable.getStartTime(output));
+ result.setExecEndTime(AbstractExecutable.getEndTime(output));
+ result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
+ for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
+ AbstractExecutable task = cubeJob.getTasks().get(i);
+ result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
+ }
+ return result;
+ }
+
+ public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
+ Preconditions.checkNotNull(stepOutput);
+ JobInstance.JobStep result = new JobInstance.JobStep();
+ result.setId(task.getId());
+ result.setName(task.getName());
+ result.setSequenceID(i);
+ result.setStatus(parseToJobStepStatus(stepOutput.getState()));
+ for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
+ if (entry.getKey() != null && entry.getValue() != null) {
+ result.putInfo(entry.getKey(), entry.getValue());
+ }
+ }
+ result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
+ result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
+ if (task instanceof ShellExecutable) {
+ result.setExecCmd(((ShellExecutable) task).getCmd());
+ }
+ if (task instanceof MapReduceExecutable) {
+ result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
+ result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
+ }
+ if (task instanceof HadoopShellExecutable) {
+ result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
+ }
+ return result;
+ }
+
+ public static JobStatusEnum parseToJobStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStatusEnum.PENDING;
+ case RUNNING:
+ return JobStatusEnum.RUNNING;
+ case ERROR:
+ return JobStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+
+ public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
+ switch (state) {
+ case READY:
+ return JobStepStatusEnum.PENDING;
+ case RUNNING:
+ return JobStepStatusEnum.RUNNING;
+ case ERROR:
+ return JobStepStatusEnum.ERROR;
+ case DISCARDED:
+ return JobStepStatusEnum.DISCARDED;
+ case SUCCEED:
+ return JobStepStatusEnum.FINISHED;
+ case STOPPED:
+ return JobStepStatusEnum.STOPPED;
+ default:
+ throw new RuntimeException("invalid state:" + state);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1e91b43..4ba426e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -37,6 +37,7 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.engine.mr.common.JobInfoConverter;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.Scheduler;
@@ -55,7 +56,6 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.util.JobInfoConverter;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
http://git-wip-us.apache.org/repos/asf/kylin/blob/598b32ef/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java b/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
deleted file mode 100644
index e13c9c3..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/JobInfoConverter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import java.util.Map;
-
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.JobInstance;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.JobStatusEnum;
-import org.apache.kylin.job.constant.JobStepStatusEnum;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.Output;
-
-import com.google.common.base.Preconditions;
-
-public class JobInfoConverter {
- public static JobInstance parseToJobInstance(AbstractExecutable job, Map<String, Output> outputs) {
- if (job == null) {
- return null;
- }
- Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
- CubingJob cubeJob = (CubingJob) job;
- Output output = outputs.get(job.getId());
- final JobInstance result = new JobInstance();
- result.setName(job.getName());
- result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
- result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
- result.setLastModified(output.getLastModified());
- result.setSubmitter(cubeJob.getSubmitter());
- result.setUuid(cubeJob.getId());
- result.setType(CubeBuildTypeEnum.BUILD);
- result.setStatus(parseToJobStatus(output.getState()));
- result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- result.setExecStartTime(AbstractExecutable.getStartTime(output));
- result.setExecEndTime(AbstractExecutable.getEndTime(output));
- result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime()) / 1000);
- for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
- AbstractExecutable task = cubeJob.getTasks().get(i);
- result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
- }
- return result;
- }
-
- public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
- Preconditions.checkNotNull(stepOutput);
- JobInstance.JobStep result = new JobInstance.JobStep();
- result.setId(task.getId());
- result.setName(task.getName());
- result.setSequenceID(i);
- result.setStatus(parseToJobStepStatus(stepOutput.getState()));
- for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
- if (entry.getKey() != null && entry.getValue() != null) {
- result.putInfo(entry.getKey(), entry.getValue());
- }
- }
- result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
- result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
- if (task instanceof ShellExecutable) {
- result.setExecCmd(((ShellExecutable) task).getCmd());
- }
- if (task instanceof MapReduceExecutable) {
- result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
- result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
- }
- if (task instanceof HadoopShellExecutable) {
- result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
- }
- return result;
- }
-
- public static JobStatusEnum parseToJobStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStatusEnum.PENDING;
- case RUNNING:
- return JobStatusEnum.RUNNING;
- case ERROR:
- return JobStatusEnum.ERROR;
- case DISCARDED:
- return JobStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStatusEnum.FINISHED;
- case STOPPED:
- return JobStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-
- public static JobStepStatusEnum parseToJobStepStatus(ExecutableState state) {
- switch (state) {
- case READY:
- return JobStepStatusEnum.PENDING;
- case RUNNING:
- return JobStepStatusEnum.RUNNING;
- case ERROR:
- return JobStepStatusEnum.ERROR;
- case DISCARDED:
- return JobStepStatusEnum.DISCARDED;
- case SUCCEED:
- return JobStepStatusEnum.FINISHED;
- case STOPPED:
- return JobStepStatusEnum.STOPPED;
- default:
- throw new RuntimeException("invalid state:" + state);
- }
- }
-}
[13/35] kylin git commit: KYLIN-2501 Stream Aggregate GTRecords at
Query Server
Posted by li...@apache.org.
KYLIN-2501 Stream Aggregate GTRecords at Query Server
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fa57248
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fa57248
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fa57248
Branch: refs/heads/master-hbase0.98
Commit: 0fa572482f209d25cca6968812d293c08d077210
Parents: fa3ee3f
Author: gaodayue <ga...@meituan.com>
Authored: Wed Mar 15 22:45:02 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 16:39:35 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../kylin/common/util/ImmutableBitSet.java | 29 ++-
.../org/apache/kylin/GTForwardingScanner.java | 56 +++++
.../kylin/cube/gridtable/CubeGridTable.java | 18 --
.../gridtable/CuboidToGridTableMapping.java | 18 ++
.../cube/inmemcubing/InMemCubeBuilder.java | 6 +-
.../kylin/gridtable/GTAggregateScanner.java | 16 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 22 +-
.../org/apache/kylin/gridtable/GTRecord.java | 80 +++----
.../apache/kylin/gridtable/GTScanRequest.java | 13 ++
.../gridtable/GTStreamAggregateScanner.java | 211 +++++++++++++++++++
.../kylin/gridtable/GTScanReqSerDerTest.java | 4 +-
.../apache/kylin/storage/StorageContext.java | 20 ++
.../storage/gtrecord/CubeScanRangePlanner.java | 3 +-
.../storage/gtrecord/CubeSegmentScanner.java | 7 +-
.../storage/gtrecord/CubeTupleConverter.java | 31 +--
.../gtrecord/GTCubeStorageQueryBase.java | 38 +++-
.../kylin/storage/gtrecord/ITupleConverter.java | 3 +-
.../gtrecord/PartitionResultIterator.java | 59 ++++++
.../storage/gtrecord/PartitionResultMerger.java | 100 +++++++++
.../kylin/storage/gtrecord/ScannerWorker.java | 5 +-
.../gtrecord/SegmentCubeTupleIterator.java | 72 ++++++-
.../gtrecord/StorageResponseGTScatter.java | 82 +++----
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 5 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 5 +-
26 files changed, 704 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02349ad..9cd35c8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -802,6 +802,10 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
+ public boolean isStreamAggregateEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.stream-aggregate-enabled", "true"));
+ }
+
@Deprecated //Limit is good even it's large. This config is meaning less since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
index b417877..5cdf08c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ImmutableBitSet.java
@@ -19,8 +19,9 @@ package org.apache.kylin.common.util;
import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Iterator;
-public class ImmutableBitSet {
+public class ImmutableBitSet implements Iterable<Integer> {
public static final ImmutableBitSet EMPTY = new ImmutableBitSet(new BitSet());
@@ -168,4 +169,30 @@ public class ImmutableBitSet {
return new ImmutableBitSet(bitSet);
}
};
+
+ /**
+ * Iterate over the positions of true value.
+ * @return the iterator
+ */
+ @Override
+ public Iterator<Integer> iterator() {
+ return new Iterator<Integer>() {
+ int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < arr.length;
+ }
+
+ @Override
+ public Integer next() {
+ return arr[index++];
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/GTForwardingScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/GTForwardingScanner.java b/core-cube/src/main/java/org/apache/kylin/GTForwardingScanner.java
new file mode 100644
index 0000000..de8c88d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/GTForwardingScanner.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin;
+
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.IGTScanner;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A {@link IGTScanner} which forwards all its method calls to another scanner.
+ *
+ * @see <a href="http://en.wikipedia.org/wiki/Decorator_pattern">decorator pattern</a>.
+ */
+public class GTForwardingScanner implements IGTScanner {
+ protected IGTScanner delegated;
+
+ protected GTForwardingScanner(IGTScanner delegated) {
+ this.delegated = checkNotNull(delegated, "delegated");
+ }
+
+ @Override
+ public GTInfo getInfo() {
+ return delegated.getInfo();
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegated.close();
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return delegated.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 563cf43..5cee9df 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -18,29 +18,11 @@
package org.apache.kylin.cube.gridtable;
-import java.util.Map;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.CubeDimEncMap;
-import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.metadata.model.TblColRef;
public class CubeGridTable {
-
- public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
- Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
- return newGTInfo(cuboid, new CubeDimEncMap(cubeSeg));
- }
-
- public static GTInfo newGTInfo(CubeDesc cubeDesc, long cuboidId, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
- return newGTInfo(cuboid, new CubeDimEncMap(cubeDesc, dictionaryMap));
- }
-
public static GTInfo newGTInfo(Cuboid cuboid, IDimensionEncodingMap dimEncMap) {
CuboidToGridTableMapping mapping = new CuboidToGridTableMapping(cuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 2e5dd12..6879687 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -140,11 +140,29 @@ public class CuboidToGridTableMapping {
return i == null ? -1 : i.intValue();
}
+ public int[] getDimIndexes(Collection<TblColRef> dims) {
+ int[] result = new int[dims.size()];
+ int i = 0;
+ for (TblColRef dim : dims) {
+ result[i++] = getIndexOf(dim);
+ }
+ return result;
+ }
+
public int getIndexOf(FunctionDesc metric) {
Integer r = metrics2gt.get(metric);
return r == null ? -1 : r;
}
+ public int[] getMetricsIndexes(Collection<FunctionDesc> metrics) {
+ int[] result = new int[metrics.size()];
+ int i = 0;
+ for (FunctionDesc metric : metrics) {
+ result[i++] = getIndexOf(metric);
+ }
+ return result;
+ }
+
public List<TblColRef> getCuboidDimensionsInGTOrder() {
return cuboid.getColumns();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index e08844e..a26e948 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -38,6 +38,7 @@ import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.gridtable.CubeGridTable;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.gridtable.GTAggregateScanner;
import org.apache.kylin.gridtable.GTBuilder;
@@ -108,7 +109,10 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private GridTable newGridTableByCuboidID(long cuboidID) throws IOException {
- GTInfo info = CubeGridTable.newGTInfo(cubeDesc, cuboidID, dictionaryMap);
+ GTInfo info = CubeGridTable.newGTInfo(
+ Cuboid.findById(cubeDesc, cuboidID),
+ new CubeDimEncMap(cubeDesc, dictionaryMap)
+ );
// Below several store implementation are very similar in performance. The ConcurrentDiskStore is the simplest.
// MemDiskStore store = new MemDiskStore(info, memBudget == null ? MemoryBudgetController.ZERO_BUDGET : memBudget);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 7cdd4f5..0dd6fa9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -45,7 +45,6 @@ import org.apache.kylin.common.util.MemoryBudgetController.MemoryWaterLevel;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureAggregators;
-import org.apache.kylin.metadata.datatype.DataType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +62,7 @@ public class GTAggregateScanner implements IGTScanner {
final ImmutableBitSet metrics;
final String[] metricsAggrFuncs;
final IGTScanner inputScanner;
+ final BufferedMeasureCodec measureCodec;
final AggregationCache aggrCache;
final long spillThreshold; // 0 means no memory control && no spill
final int storagePushDownLimit;//default to be Int.MAX
@@ -86,6 +86,7 @@ public class GTAggregateScanner implements IGTScanner {
this.metrics = req.getAggrMetrics();
this.metricsAggrFuncs = req.getAggrMetricsFuncs();
this.inputScanner = inputScanner;
+ this.measureCodec = req.createMeasureCodec();
this.aggrCache = new AggregationCache();
this.spillThreshold = (long) (req.getAggCacheMemThreshold() * MemoryBudgetController.ONE_GB);
this.aggrMask = new boolean[metricsAggrFuncs.length];
@@ -175,7 +176,6 @@ public class GTAggregateScanner implements IGTScanner {
final int keyLength;
final boolean[] compareMask;
boolean compareAll = true;
- final BufferedMeasureCodec measureCodec;
final Comparator<byte[]> bytesComparator = new Comparator<byte[]>() {
@Override
@@ -213,18 +213,6 @@ public class GTAggregateScanner implements IGTScanner {
keyLength = compareMask.length;
dumps = Lists.newArrayList();
aggBufMap = createBuffMap();
- measureCodec = createMeasureCodec();
- }
-
- private BufferedMeasureCodec createMeasureCodec() {
- DataType[] types = new DataType[metrics.trueBitCount()];
- for (int i = 0; i < types.length; i++) {
- types[i] = info.getColumnType(metrics.trueBitAt(i));
- }
-
- BufferedMeasureCodec result = new BufferedMeasureCodec(types);
- result.setBufferSize(info.getMaxColumnLength(metrics));
- return result;
}
private boolean[] createCompareMask() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
index 717f89c..cad0a04 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTFilterScanner.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.kylin.GTForwardingScanner;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -33,17 +34,16 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-public class GTFilterScanner implements IGTScanner {
+public class GTFilterScanner extends GTForwardingScanner {
- final private IGTScanner inputScanner;
final private TupleFilter filter;
final private IFilterCodeSystem<ByteArray> filterCodeSystem;
final private IEvaluatableTuple oneTuple; // avoid instance creation
private GTRecord next = null;
- public GTFilterScanner(IGTScanner inputScanner, GTScanRequest req) throws IOException {
- this.inputScanner = inputScanner;
+ public GTFilterScanner(IGTScanner delegated, GTScanRequest req) throws IOException {
+ super(delegated);
this.filter = req.getFilterPushDown();
this.filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator());
this.oneTuple = new IEvaluatableTuple() {
@@ -53,25 +53,15 @@ public class GTFilterScanner implements IGTScanner {
}
};
- if (TupleFilter.isEvaluableRecursively(filter) == false)
+ if (!TupleFilter.isEvaluableRecursively(filter))
throw new IllegalArgumentException();
}
@Override
- public GTInfo getInfo() {
- return inputScanner.getInfo();
- }
-
- @Override
- public void close() throws IOException {
- inputScanner.close();
- }
-
- @Override
public Iterator<GTRecord> iterator() {
return new Iterator<GTRecord>() {
- private Iterator<GTRecord> inputIterator = inputScanner.iterator();
+ private Iterator<GTRecord> inputIterator = delegated.iterator();
private FilterResultCache resultCache = new FilterResultCache(getInfo(), filter);
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index f4480c8..3397adc 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -21,7 +21,6 @@ package org.apache.kylin.gridtable;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Comparator;
-import java.util.List;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -46,18 +45,21 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
}
this.info = info;
}
-
- public GTRecord(GTRecord other) {
- this.info = other.info;
- this.cols = new ByteArray[info.getColumnCount()];
- for (int i = 0; i < other.cols.length; i++) {
- this.cols[i] = other.cols[i].copy();
+
+ @Override
+ public GTRecord clone() { // deep copy
+ ByteArray[] cols = new ByteArray[this.cols.length];
+ for (int i = 0; i < cols.length; i++) {
+ cols[i] = this.cols[i].copy();
}
+ return new GTRecord(this.info, cols);
}
- @Override
- public Object clone() {
- return new GTRecord(this);
+ public void shallowCopyFrom(GTRecord source) {
+ assert info == source.info;
+ for (int i = 0; i < cols.length; i++) {
+ cols[i].set(source.cols[i]);
+ }
}
public GTInfo getInfo() {
@@ -106,30 +108,18 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
/** decode and return the values of this record */
public Object[] getValues(ImmutableBitSet selectedCols, Object[] result) {
assert selectedCols.cardinality() == result.length;
-
for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- if (cols[c] == null || cols[c].array() == null) {
- result[i] = null;
- } else {
- result[i] = info.codeSystem.decodeColumnValue(c, cols[c].asBuffer());
- }
+ result[i] = decodeValue(selectedCols.trueBitAt(i));
}
return result;
}
- /** decode and return the values of this record */
- public Object[] getValues(int[] selectedColumns, Object[] result) {
- assert selectedColumns.length <= result.length;
- for (int i = 0; i < selectedColumns.length; i++) {
- int c = selectedColumns[i];
- if (cols[c].array() == null) {
- result[i] = null;
- } else {
- result[i] = info.codeSystem.decodeColumnValue(c, cols[c].asBuffer());
- }
+ public Object decodeValue(int c) {
+ ByteArray col = cols[c];
+ if (col != null && col.array() != null) {
+ return info.codeSystem.decodeColumnValue(c, col.asBuffer());
}
- return result;
+ return null;
}
public int sizeOf(ImmutableBitSet selectedCols) {
@@ -198,19 +188,13 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
return compareToInternal(o, info.colAll);
}
- public int compareToOnPrimaryKey(GTRecord o) {
- return compareToInternal(o, info.primaryKey);
- }
-
- public static Comparator<GTRecord> getPrimaryKeyComparator() {
+ public static Comparator<GTRecord> getComparator(final ImmutableBitSet participateCols) {
return new Comparator<GTRecord>() {
- @Override
public int compare(GTRecord o1, GTRecord o2) {
if (o1 == null || o2 == null) {
throw new IllegalStateException("Cannot handle null");
}
-
- return o1.compareToOnPrimaryKey(o2);
+ return o1.compareToInternal(o2, participateCols);
}
};
}
@@ -287,26 +271,14 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable {
loadColumns(info.colBlocks[c], buf);
}
- /** change pointers to point to data in given buffer, UNLIKE deserialize */
- public void loadColumns(ImmutableBitSet selectedCols, ByteBuffer buf) {
- int pos = buf.position();
- for (int i = 0; i < selectedCols.trueBitCount(); i++) {
- int c = selectedCols.trueBitAt(i);
- int len = info.codeSystem.codeLength(c, buf);
- cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
- pos += len;
- buf.position(pos);
- }
- }
-
- /** change pointers to point to data in given buffer, UNLIKE deserialize
- * unlike loadColumns(ImmutableBitSet selectedCols, ByteBuffer buf), this
- * method allows to defined specific columns(in order) to load
+ /**
+ * Change pointers to point to data in given buffer, UNLIKE deserialize
+ * @param selectedCols positions of column to load
+ * @param buf buffer containing continuous data of selected columns
*/
- public void loadColumns(List<Integer> selectedCols, ByteBuffer buf) {
+ public void loadColumns(Iterable<Integer> selectedCols, ByteBuffer buf) {
int pos = buf.position();
- for (int i = 0; i < selectedCols.size(); i++) {
- int c = selectedCols.get(i);
+ for (int c : selectedCols) {
int len = info.codeSystem.codeLength(c, buf);
cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
pos += len;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 4629c8e..ae35d2b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -31,6 +31,8 @@ import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.SerializeToByteBuffer;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -202,6 +204,17 @@ public class GTScanRequest {
}
+ public BufferedMeasureCodec createMeasureCodec() {
+ DataType[] metricTypes = new DataType[aggrMetrics.trueBitCount()];
+ for (int i = 0; i < metricTypes.length; i++) {
+ metricTypes[i] = info.getColumnType(aggrMetrics.trueBitAt(i));
+ }
+
+ BufferedMeasureCodec codec = new BufferedMeasureCodec(metricTypes);
+ codec.setBufferSize(info.getMaxColumnLength(aggrMetrics));
+ return codec;
+ }
+
public boolean isDoingStorageAggregation() {
return doingStorageAggregation;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
new file mode 100644
index 0000000..1fde423
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.gridtable;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import org.apache.kylin.GTForwardingScanner;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.measure.BufferedMeasureCodec;
+import org.apache.kylin.measure.MeasureAggregator;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * GTStreamAggregateScanner requires input records to be sorted on group fields.
+ * In such cases, it's superior to hash/sort based aggregator because it can produce
+ * ordered outputs on the fly and the memory consumption is very low.
+ */
+public class GTStreamAggregateScanner extends GTForwardingScanner {
+ private final GTScanRequest req;
+ private final Comparator<GTRecord> keyComparator;
+
+ public GTStreamAggregateScanner(IGTScanner delegated,
+ GTScanRequest req, Comparator<GTRecord> keyComparator) {
+ super(delegated);
+ this.req = req;
+ this.keyComparator = keyComparator;
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ return new StreamMergeGTRecordIterator(delegated.iterator());
+ }
+
+ public Iterator<Object[]> valuesIterator(int[] gtDimsIdx, int[] gtMetricsIdx) {
+ return new StreamMergeValuesIterator(delegated.iterator(), gtDimsIdx, gtMetricsIdx);
+ }
+
+ private abstract class AbstractStreamMergeIterator<E> implements Iterator<E> {
+ final PeekingIterator<GTRecord> input;
+ final IGTCodeSystem codeSystem;
+ final ImmutableBitSet dimensions;
+ final ImmutableBitSet metrics;
+ final String[] metricFuncs;
+ final BufferedMeasureCodec measureCodec;
+
+ private final GTRecord first; // reuse to avoid object creation
+
+ AbstractStreamMergeIterator(Iterator<GTRecord> input) {
+ this.input = Iterators.peekingIterator(input);
+ this.codeSystem = req.getInfo().getCodeSystem();
+ this.dimensions = req.getDimensions();
+ this.metrics = req.getAggrMetrics();
+ this.metricFuncs = req.getAggrMetricsFuncs();
+ this.measureCodec = req.createMeasureCodec();
+
+ this.first = new GTRecord(req.getInfo());
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ private boolean isSameKey(GTRecord o1, GTRecord o2) {
+ return keyComparator.compare(o1, o2) == 0;
+ }
+
+ private boolean shouldMergeNext(GTRecord current) {
+ return input.hasNext() && isSameKey(current, input.peek());
+ }
+
+ protected abstract E finalizeResult(GTRecord record);
+
+ protected abstract E finalizeResult(GTRecord record, Object[] aggStates);
+
+ @Override
+ public E next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ // WATCH OUT! record returned by "input" scanner could be changed later,
+ // so we must make a shallow copy of it.
+ first.shallowCopyFrom(input.next());
+
+ // shortcut to avoid extra deserialize/serialize cost
+ if (!shouldMergeNext(first)) {
+ return finalizeResult(first);
+ }
+ // merge records with the same key
+ MeasureAggregator[] aggrs = codeSystem.newMetricsAggregators(metrics, metricFuncs);
+ aggregate(aggrs, first);
+ aggregate(aggrs, input.next()); // no need to copy record because it's not referred to later
+ while (shouldMergeNext(first)) {
+ aggregate(aggrs, input.next());
+ }
+
+ Object[] aggStates = new Object[aggrs.length];
+ for (int i = 0; i < aggStates.length; i++) {
+ aggStates[i] = aggrs[i].getState();
+ }
+ return finalizeResult(first, aggStates);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void aggregate(MeasureAggregator[] aggregators, GTRecord record) {
+ for (int i = 0; i < aggregators.length; i++) {
+ int c = metrics.trueBitAt(i);
+ Object metric = codeSystem.decodeColumnValue(c, record.cols[c].asBuffer());
+ aggregators[i].aggregate(metric);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("remove");
+ }
+ }
+
+ private class StreamMergeGTRecordIterator extends AbstractStreamMergeIterator<GTRecord> {
+
+ private GTRecord returnRecord; // avoid object creation
+
+ StreamMergeGTRecordIterator(Iterator<GTRecord> input) {
+ super(input);
+ this.returnRecord = new GTRecord(req.getInfo());
+ }
+
+ @Override
+ protected GTRecord finalizeResult(GTRecord record) {
+ return record;
+ }
+
+ @Override
+ protected GTRecord finalizeResult(GTRecord record, Object[] aggStates) {
+ // 1. load dimensions
+ for (int c : dimensions) {
+ returnRecord.cols[c] = record.cols[c];
+ }
+ // 2. serialize metrics
+ byte[] bytes = measureCodec.encode(aggStates).array();
+ int[] sizes = measureCodec.getMeasureSizes();
+ // 3. load metrics
+ int offset = 0;
+ for (int i = 0; i < metrics.trueBitCount(); i++) {
+ int c = metrics.trueBitAt(i);
+ returnRecord.cols[c].set(bytes, offset, sizes[i]);
+ offset += sizes[i];
+ }
+ return returnRecord;
+ }
+ }
+
+ private class StreamMergeValuesIterator extends AbstractStreamMergeIterator<Object[]> {
+
+ private int[] gtDimsIdx;
+ private int[] gtMetricsIdx;
+ private Object[] result; // avoid object creation
+
+ StreamMergeValuesIterator(Iterator<GTRecord> input, int[] gtDimsIdx, int[] gtMetricsIdx) {
+ super(input);
+ this.gtDimsIdx = gtDimsIdx;
+ this.gtMetricsIdx = gtMetricsIdx;
+ result = new Object[gtDimsIdx.length + gtMetricsIdx.length];
+ }
+
+ private void decodeAndSetDimensions(GTRecord record) {
+ for (int i = 0; i < gtDimsIdx.length; i++) {
+ result[i] = record.decodeValue(gtDimsIdx[i]);
+ }
+ }
+
+ @Override
+ protected Object[] finalizeResult(GTRecord record) {
+ decodeAndSetDimensions(record);
+ // decode metrics
+ for (int i = 0; i < gtMetricsIdx.length; i++) {
+ result[gtDimsIdx.length + i] = record.decodeValue(gtMetricsIdx[i]);
+ }
+ return result;
+ }
+
+ @Override
+ protected Object[] finalizeResult(GTRecord record, Object[] aggStates) {
+ decodeAndSetDimensions(record);
+ // set metrics
+ for (int i = 0; i < gtMetricsIdx.length; i++) {
+ result[gtDimsIdx.length + i] = aggStates[i];
+ }
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
index 77cc2d8..1ae229a 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
@@ -29,6 +29,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeGridTable;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -90,7 +91,8 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
CubeSegment segment = cube.getFirstSegment();
- GTInfo info = CubeGridTable.newGTInfo(segment, Cuboid.getBaseCuboidId(cube.getDescriptor()));
+ Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
+ GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
GTInfo.serializer.serialize(info, buffer);
buffer.flip();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 998f1db..bb17054 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -18,10 +18,12 @@
package org.apache.kylin.storage;
+import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.storage.gtrecord.GTCubeStorageQueryBase;
import org.slf4j.Logger;
@@ -48,6 +50,9 @@ public class StorageContext {
private boolean needStorageAggregation = false;
private boolean enableCoprocessor = false;
+ private boolean enableStreamAggregate = false;
+ private Comparator<GTRecord> groupKeyComparator;
+
private IStorageQuery storageQuery;
private AtomicLong processedRowCount = new AtomicLong();
private Cuboid cuboid;
@@ -230,4 +235,19 @@ public class StorageContext {
this.storageQuery = storageQuery;
}
+ public boolean isStreamAggregateEnabled() {
+ return enableStreamAggregate;
+ }
+
+ public void enableStreamAggregate() {
+ this.enableStreamAggregate = true;
+ }
+
+ public Comparator<GTRecord> getGroupKeyComparator() {
+ return groupKeyComparator;
+ }
+
+ public void setGroupKeyComparator(Comparator<GTRecord> groupKeyComparator) {
+ this.groupKeyComparator = groupKeyComparator;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index 6911827..c3cc858 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -38,6 +38,7 @@ import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.gridtable.RecordComparators;
import org.apache.kylin.cube.gridtable.ScanRangePlannerBase;
import org.apache.kylin.cube.gridtable.SegmentGTStartAndEnd;
+import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
@@ -85,7 +86,7 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
Set<TblColRef> filterDims = Sets.newHashSet();
TupleFilter.collectColumns(filter, filterDims);
- this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
+ this.gtInfo = CubeGridTable.newGTInfo(cuboid, new CubeDimEncMap(cubeSegment));
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 4f206d4..31a9f99 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -78,7 +78,7 @@ public class CubeSegmentScanner implements IGTScanner {
}
scanRequest = scanRangePlanner.planScanRequest();
String gtStorage = ((GTCubeStorageQueryBase) context.getStorageQuery()).getGTStorage();
- scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
+ scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage, context);
}
@Override
@@ -96,8 +96,7 @@ public class CubeSegmentScanner implements IGTScanner {
return scanRequest == null ? null : scanRequest.getInfo();
}
- public CubeSegment getSegment() {
- return this.cubeSeg;
+ public GTScanRequest getScanRequest() {
+ return scanRequest;
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 280718f..b762e5c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -28,10 +28,8 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -43,7 +41,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
- * convert GTRecord to tuple
+ * Convert Object[] (decoded GTRecord) to tuple
*/
public class CubeTupleConverter implements ITupleConverter {
@@ -54,7 +52,6 @@ public class CubeTupleConverter implements ITupleConverter {
private final int[] gtColIdx;
private final int[] tupleIdx;
- private final Object[] gtValues;
private final MeasureType<?>[] measureTypes;
private final List<IAdvMeasureFiller> advMeasureFillers;
@@ -63,19 +60,16 @@ public class CubeTupleConverter implements ITupleConverter {
private final int nSelectedDims;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
- Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
+ Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx,
+ TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
this.cuboid = cuboid;
+ this.gtColIdx = gtColIdx;
this.tupleInfo = returnTupleInfo;
this.derivedColFillers = Lists.newArrayList();
- List<TblColRef> cuboidDims = cuboid.getColumns();
- CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
-
nSelectedDims = selectedDimensions.size();
- gtColIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];
- gtValues = new Object[selectedDimensions.size() + selectedMetrics.size()];
// measure types don't have this many, but aligned length make programming easier
measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];
@@ -89,21 +83,11 @@ public class CubeTupleConverter implements ITupleConverter {
// pre-calculate dimension index mapping to tuple
for (TblColRef dim : selectedDimensions) {
- int dimIndex = mapping.getIndexOf(dim);
- gtColIdx[i] = dimIndex;
tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
-
- // if (tupleIdx[iii] == -1) {
- // throw new IllegalStateException("dim not used in tuple:" + dim);
- // }
-
i++;
}
for (FunctionDesc metric : selectedMetrics) {
- int metricIndex = mapping.getIndexOf(metric);
- gtColIdx[i] = metricIndex;
-
if (metric.needRewrite()) {
String rewriteFieldName = metric.getRewriteFieldName();
tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
@@ -126,7 +110,7 @@ public class CubeTupleConverter implements ITupleConverter {
}
// prepare derived columns and filler
- Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboidDims, null);
+ Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboid.getColumns(), null);
for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
TblColRef[] hostCols = entry.getKey().data;
for (DeriveInfo deriveInfo : entry.getValue()) {
@@ -148,9 +132,8 @@ public class CubeTupleConverter implements ITupleConverter {
}
@Override
- public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple) {
-
- record.getValues(gtColIdx, gtValues);
+ public List<IAdvMeasureFiller> translateResult(Object[] gtValues, Tuple tuple) {
+ assert gtValues.length == gtColIdx.length;
// dimensions
for (int i = 0; i < nSelectedDims; i++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ecf1ad3..82590a2 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -26,15 +26,18 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
+import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -120,6 +123,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
+ // set whether to aggregate results from multiple partitions
+ enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
// set query deadline
context.setDeadline(cubeInstance);
@@ -144,8 +149,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
protected abstract String getGTStorage();
- protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
- return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+ protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) {
+ return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo);
}
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
@@ -366,6 +371,35 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
+ private void enableStreamAggregateIfBeneficial(Cuboid cuboid, Set<TblColRef> groupsD, StorageContext context) {
+ CubeDesc cubeDesc = cuboid.getCubeDesc();
+ boolean enabled = cubeDesc.getConfig().isStreamAggregateEnabled();
+
+ Set<TblColRef> shardByInGroups = Sets.newHashSet();
+ for (TblColRef col : cubeDesc.getShardByColumns()) {
+ if (groupsD.contains(col)) {
+ shardByInGroups.add(col);
+ }
+ }
+ if (!shardByInGroups.isEmpty()) {
+ enabled = false;
+ logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: " + shardByInGroups);
+ }
+
+ if (!context.isNeedStorageAggregation()) {
+ enabled = false;
+ logger.debug("Aggregate partition results is not beneficial because no storage aggregation");
+ }
+
+ if (enabled) {
+ CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
+ ImmutableBitSet cols = mapping.makeGridTableColumns(groupsD);
+
+ context.enableStreamAggregate();
+ context.setGroupKeyComparator(GTRecord.getComparator(cols));
+ }
+ }
+
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
index 9c50d0c..dd48e4d 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ITupleConverter.java
@@ -20,11 +20,10 @@ package org.apache.kylin.storage.gtrecord;
import java.util.List;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.tuple.Tuple;
public interface ITupleConverter {
- public List<IAdvMeasureFiller> translateResult(GTRecord record, Tuple tuple);
+ public List<IAdvMeasureFiller> translateResult(Object[] gtValues, Tuple tuple);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java
new file mode 100644
index 0000000..474e1e0
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultIterator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import com.google.common.collect.UnmodifiableIterator;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.nio.ByteBuffer;
+import java.util.NoSuchElementException;
+
+/**
+ * Support iterate over {@code GTRecord}s in storage partition result.
+ *
+ * <p>Note that the implementation returns the same object for next().
+ * Client needs to copy the returned record when needed.
+ */
+public class PartitionResultIterator extends UnmodifiableIterator<GTRecord> {
+ private final ByteBuffer buffer;
+ private final ImmutableBitSet cols;
+ private final GTRecord record; // reuse to avoid object creation
+
+ public PartitionResultIterator(byte[] data, GTInfo info, ImmutableBitSet cols) {
+ this.buffer = ByteBuffer.wrap(data);
+ this.cols = cols;
+ this.record = new GTRecord(info);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return buffer.hasRemaining();
+ }
+
+ @Override
+ public GTRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ record.loadColumns(cols, buffer);
+ return record;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
new file mode 100644
index 0000000..52029d3
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import com.google.common.collect.UnmodifiableIterator;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.PriorityQueue;
+
+/**
+ * Merge-sort {@code GTRecord}s in all partitions, assume each partition contains sorted elements.
+ */
+public class PartitionResultMerger implements Iterable<GTRecord> {
+ private final ImmutableList<PartitionResultIterator> partitionResults;
+ private final GTInfo info;
+ private final Comparator<GTRecord> comparator;
+
+ public PartitionResultMerger(
+ Iterable<PartitionResultIterator> partitionResults,
+ GTInfo info, Comparator<GTRecord> comparator) {
+ this.partitionResults = ImmutableList.copyOf(partitionResults);
+ this.info = info;
+ this.comparator = comparator;
+ }
+
+ @Override
+ public Iterator<GTRecord> iterator() {
+ if (partitionResults.size() == 1) {
+ return partitionResults.get(0);
+ }
+ return new MergingResultsIterator();
+ }
+
+ private class MergingResultsIterator extends UnmodifiableIterator<GTRecord> {
+ final GTRecord record = new GTRecord(info); // reuse to avoid object creation
+
+ PriorityQueue<PeekingIterator<GTRecord>> heap;
+
+ MergingResultsIterator() {
+ Comparator<PeekingIterator<GTRecord>> heapComparator = new Comparator<PeekingIterator<GTRecord>>() {
+ public int compare(PeekingIterator<GTRecord> o1, PeekingIterator<GTRecord> o2) {
+ return comparator.compare(o1.peek(), o2.peek());
+ }
+ };
+ this.heap = new PriorityQueue<>(partitionResults.size(), heapComparator);
+
+ for (PartitionResultIterator it : partitionResults) {
+ if (it.hasNext()) {
+ heap.offer(Iterators.peekingIterator(it));
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !heap.isEmpty();
+ }
+
+ @Override
+ public GTRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ // get smallest record
+ PeekingIterator<GTRecord> it = heap.poll();
+ // WATCH OUT! record got from PartitionResultIterator.next() may changed later,
+ // so we must make a shallow copy of it.
+ record.shallowCopyFrom(it.next());
+
+ if (it.hasNext()) {
+ heap.offer(it);
+ }
+
+ return record;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
index 9e89227..fe22e9c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -30,6 +30,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStorage;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +39,7 @@ public class ScannerWorker {
private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
private IGTScanner internal = null;
- public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
+ public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage, StorageContext context) {
if (scanRequest == null) {
logger.info("Segment {} will be skipped", segment);
internal = new EmptyGTScanner();
@@ -48,7 +49,7 @@ public class ScannerWorker {
final GTInfo info = scanRequest.getInfo();
try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class).newInstance(segment, cuboid, info); // default behavior
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class).newInstance(segment, cuboid, info, context); // default behavior
internal = rpc.getGTScanner(scanRequest);
} catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 37699a3..11f766c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -24,8 +24,14 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import com.google.common.collect.UnmodifiableIterator;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
+import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTStreamAggregateScanner;
+import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -49,7 +55,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
protected final Tuple tuple;
protected final StorageContext context;
- protected Iterator<GTRecord> gtItr;
+ protected Iterator<Object[]> gtValues;
protected ITupleConverter cubeTupleConverter;
protected Tuple next;
@@ -66,12 +72,62 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
this.tupleInfo = returnTupleInfo;
this.tuple = new Tuple(returnTupleInfo);
this.context = context;
- this.gtItr = getGTItr(scanner);
- this.cubeTupleConverter = ((GTCubeStorageQueryBase) context.getStorageQuery()).newCubeTupleConverter(scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
+
+ CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
+ int[] gtDimsIdx = mapping.getDimIndexes(selectedDimensions);
+ int[] gtMetricsIdx = mapping.getMetricsIndexes(selectedMetrics);
+ // gtColIdx = gtDimsIdx + gtMetricsIdx
+ int[] gtColIdx = new int[gtDimsIdx.length + gtMetricsIdx.length];
+ System.arraycopy(gtDimsIdx, 0, gtColIdx, 0, gtDimsIdx.length);
+ System.arraycopy(gtMetricsIdx, 0, gtColIdx, gtDimsIdx.length, gtMetricsIdx.length);
+
+ this.gtValues = getGTValuesIterator(scanner.iterator(), scanner.getScanRequest(), gtDimsIdx, gtMetricsIdx);
+ this.cubeTupleConverter = ((GTCubeStorageQueryBase) context.getStorageQuery()).newCubeTupleConverter(
+ scanner.cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo);
}
- private Iterator<GTRecord> getGTItr(CubeSegmentScanner scanner) {
- return scanner.iterator();
+ private Iterator<Object[]> getGTValuesIterator(
+ final Iterator<GTRecord> records, final GTScanRequest scanRequest,
+ final int[] gtDimsIdx, final int[] gtMetricsIdx) {
+
+ boolean singlePartitionResult = records instanceof PartitionResultIterator;
+ if (context.isStreamAggregateEnabled() && !singlePartitionResult) {
+ // input records are ordered, leverage stream aggregator to produce possibly fewer records
+ IGTScanner inputScanner = new IGTScanner() {
+ public GTInfo getInfo() {
+ return scanRequest.getInfo();
+ }
+
+ public void close() throws IOException {}
+
+ public Iterator<GTRecord> iterator() {
+ return records;
+ }
+ };
+ GTStreamAggregateScanner aggregator = new GTStreamAggregateScanner(
+ inputScanner, scanRequest, context.getGroupKeyComparator());
+ return aggregator.valuesIterator(gtDimsIdx, gtMetricsIdx);
+ }
+
+ // simply decode records
+ return new UnmodifiableIterator<Object[]>() {
+ Object[] result = new Object[gtDimsIdx.length + gtMetricsIdx.length];
+
+ public boolean hasNext() {
+ return records.hasNext();
+ }
+
+ public Object[] next() {
+ GTRecord record = records.next();
+ for (int i = 0; i < gtDimsIdx.length; i++) {
+ result[i] = record.decodeValue(gtDimsIdx[i]);
+ }
+ for (int i = 0; i < gtMetricsIdx.length; i++) {
+ result[gtDimsIdx.length + i] = record.decodeValue(gtMetricsIdx[i]);
+ }
+ return result;
+ }
+ };
}
@Override
@@ -91,13 +147,13 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
}
// now we have a GTRecord
- if (!gtItr.hasNext()) {
+ if (!gtValues.hasNext()) {
return false;
}
- GTRecord curRecord = gtItr.next();
+ Object[] gtValues = this.gtValues.next();
// translate into tuple
- advMeasureFillers = cubeTupleConverter.translateResult(curRecord, tuple);
+ advMeasureFillers = cubeTupleConverter.translateResult(gtValues, tuple);
// the simple case
if (advMeasureFillers == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 1a80bbf..0f1e191 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -18,22 +18,20 @@
package org.apache.kylin.storage.gtrecord;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-
-import javax.annotation.Nullable;
-
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.storage.StorageContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
/**
* scatter the blob returned from region server to a iterable of gtrecords
@@ -42,18 +40,20 @@ public class StorageResponseGTScatter implements IGTScanner {
private static final Logger logger = LoggerFactory.getLogger(StorageResponseGTScatter.class);
- private GTInfo info;
+ private final GTInfo info;
private IPartitionStreamer partitionStreamer;
- private Iterator<byte[]> blocks;
- private ImmutableBitSet columns;
- private int storagePushDownLimit = -1;
+ private final Iterator<byte[]> blocks;
+ private final ImmutableBitSet columns;
+ private final StorageContext context;
+ private final boolean needSorted; // whether scanner should return sorted records
- public StorageResponseGTScatter(GTInfo info, IPartitionStreamer partitionStreamer, ImmutableBitSet columns, int storagePushDownLimit) {
- this.info = info;
+ public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, StorageContext context) {
+ this.info = scanRequest.getInfo();
this.partitionStreamer = partitionStreamer;
this.blocks = partitionStreamer.asByteArrayIterator();
- this.columns = columns;
- this.storagePushDownLimit = storagePushDownLimit;
+ this.columns = scanRequest.getColumns();
+ this.context = context;
+ this.needSorted = (context.getFinalPushDownLimit() != Integer.MAX_VALUE) || context.isStreamAggregateEnabled();
}
@Override
@@ -69,48 +69,18 @@ public class StorageResponseGTScatter implements IGTScanner {
@Override
public Iterator<GTRecord> iterator() {
- Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks, new EndpointResponseGTScatterFunc());
- if (StorageContext.mergeSortPartitionResults(storagePushDownLimit)) {
- logger.info("Using SortedIteratorMergerWithLimit to merge partition results");
- return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit, GTRecord.getPrimaryKeyComparator()).getIterator();
- } else {
- logger.info("Using Iterators.concat to merge partition results");
- return Iterators.concat(shardSubsets);
+ List<PartitionResultIterator> partitionResults = Lists.newArrayList();
+ while (blocks.hasNext()) {
+ partitionResults.add(new PartitionResultIterator(blocks.next(), info, columns));
}
- }
-
- class EndpointResponseGTScatterFunc implements Function<byte[], Iterator<GTRecord>> {
- @Nullable
- @Override
- public Iterator<GTRecord> apply(@Nullable final byte[] input) {
-
- return new Iterator<GTRecord>() {
- private ByteBuffer inputBuffer = null;
- //rotate between two buffer GTRecord to support SortedIteratorMergerWithLimit, which will peek one more GTRecord
- private GTRecord firstRecord = null;
-
- @Override
- public boolean hasNext() {
- if (inputBuffer == null) {
- inputBuffer = ByteBuffer.wrap(input);
- firstRecord = new GTRecord(info);
- }
- return inputBuffer.position() < inputBuffer.limit();
- }
-
- @Override
- public GTRecord next() {
- firstRecord.loadColumns(columns, inputBuffer);
- return firstRecord;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ if (!needSorted) {
+ logger.debug("Using Iterators.concat to merge partition results");
+ return Iterators.concat(partitionResults.iterator());
}
- }
+ logger.debug("Using PartitionResultMerger to merge partition results");
+ PartitionResultMerger merger = new PartitionResultMerger(partitionResults, info, context.getGroupKeyComparator());
+ return merger.iterator();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 82b67b6..e822ada 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -47,6 +47,7 @@ import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.gtrecord.DummyPartitionStreamer;
import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -69,8 +70,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private static ExecutorService executorService = new LoggableCachedThreadPool();
- public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
- super(segment, cuboid, fullGTInfo);
+ public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo, StorageContext context) {
+ super(segment, cuboid, fullGTInfo, context);
}
private byte[] getByteArrayForShort(short v) {
@@ -245,7 +246,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
});
}
- return new StorageResponseGTScatter(fullGTInfo, new DummyPartitionStreamer(epResultItr), scanRequest.getColumns(), scanRequest.getStoragePushDownLimit());
+ return new StorageResponseGTScatter(scanRequest, new DummyPartitionStreamer(epResultItr), storageContext);
}
private ByteString serializeGTScanReq(GTScanRequest scanRequest) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 88e7176..db81646 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -49,6 +49,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
import org.apache.kylin.gridtable.IGTStorage;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,17 +65,19 @@ public abstract class CubeHBaseRPC implements IGTStorage {
final protected Cuboid cuboid;
final protected GTInfo fullGTInfo;
final protected QueryContext queryContext;
+ final protected StorageContext storageContext;
final private RowKeyEncoder fuzzyKeyEncoder;
final private RowKeyEncoder fuzzyMaskEncoder;
- public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo, StorageContext context) {
Preconditions.checkArgument(segment instanceof CubeSegment, "segment must be CubeSegment");
this.cubeSeg = (CubeSegment) segment;
this.cuboid = cuboid;
this.fullGTInfo = fullGTInfo;
this.queryContext = QueryContext.current();
+ this.storageContext = context;
this.fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
this.fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fa57248/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 33f8d90..951e2ef 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -42,6 +42,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,8 +88,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
}
- public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo) {
- super(segment, cuboid, fullGTInfo);
+ public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo, StorageContext context) {
+ super(segment, cuboid, fullGTInfo, context);
}
@Override
[08/35] kylin git commit: minor,
set default cubing algorithm to layer for stableness
Posted by li...@apache.org.
minor, set default cubing algorithm to layer for stableness
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a776ef67
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a776ef67
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a776ef67
Branch: refs/heads/master-hbase0.98
Commit: a776ef6747638ab757c9a5298b500dd18756f5b5
Parents: 4c21821
Author: Li Yang <li...@apache.org>
Authored: Wed Mar 29 11:51:22 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Mar 29 11:57:24 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a776ef67/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1a55c94..d39f7ee 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -144,7 +144,7 @@ kylin.engine.mr.uhc-reducer-count=1
### CUBE | DICTIONARY ###
# 'auto', 'inmem' or 'layer'
-kylin.cube.algorithm=auto
+kylin.cube.algorithm=layer
# A smaller threshold prefers layer, a larger threshold prefers in-mem
kylin.cube.algorithm.layer-or-inmem-threshold=7
[15/35] kylin git commit: KYLIN-2521 upgrade calcite to 1.12
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0e41dcf..40ccd0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,7 +66,7 @@
<!-- <reflections.version>0.9.10</reflections.version> -->
<!-- Calcite Version -->
- <calcite.version>1.11.0</calcite.version>
+ <calcite.version>1.12.0</calcite.version>
<avatica.version>1.9.0</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
index 03a0674..cdd6004 100644
--- a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
+++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java
@@ -100,7 +100,7 @@ public class AggregateMultipleExpandRule extends RelOptRule {
rexNodes.add(rexBuilder.makeInputRef(type, index++));
groupKey = groupKeyIter.next();
} else {
- rexNodes.add(rexBuilder.makeNullLiteral(type.getSqlTypeName()));
+ rexNodes.add(rexBuilder.makeNullLiteral(type));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index f7877be..35dd73f 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -175,7 +175,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
planner.removeRule(DateRangeRules.FILTER_INSTANCE);
- planner.removeRule(SemiJoinRule.INSTANCE);
+ planner.removeRule(SemiJoinRule.JOIN);
+ planner.removeRule(SemiJoinRule.PROJECT);
// distinct count will be split into a separated query that is joined with the left query
planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 0468c91..4a49d04 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -74,6 +74,12 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
@Override
public Result implement(EnumerableRelImplementor enumImplementor, Prefer pref) {
+ if (System.getProperty("calcite.debug") != null) {
+ String dumpPlan = RelOptUtil.dumpPlan("", this, false, SqlExplainLevel.DIGEST_ATTRIBUTES);
+ System.out.println("EXECUTION PLAN BEFORE REWRITE");
+ System.out.println(dumpPlan);
+ }
+
// post-order travel children
OLAPRel.OLAPImplementor olapImplementor = new OLAPRel.OLAPImplementor();
olapImplementor.visitChild(getInput(), this);
[24/35] kylin git commit: KYLIN-2407 fix OLAPTableScan choose exec
function issue
Posted by li...@apache.org.
KYLIN-2407 fix OLAPTableScan choose exec function issue
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c85a0e73
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c85a0e73
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c85a0e73
Branch: refs/heads/master-hbase0.98
Commit: c85a0e73192ea65d5bf12d116a0f791261365753
Parents: 4ddcc69
Author: etherge <et...@163.com>
Authored: Wed Feb 15 13:30:44 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Apr 2 09:15:10 2017 +0800
----------------------------------------------------------------------
.../resources/query/sql_subquery/query15.sql | 29 ++++++++++++++++++++
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
2 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c85a0e73/kylin-it/src/test/resources/query/sql_subquery/query15.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query15.sql b/kylin-it/src/test/resources/query/sql_subquery/query15.sql
new file mode 100644
index 0000000..330dc27
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query15.sql
@@ -0,0 +1,29 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ week_beg_dt
+FROM
+ edw.test_cal_dt t1
+WHERE
+ t1.cal_dt IN (SELECT
+ cal_dt
+ FROM
+ test_kylin_fact
+ WHERE
+ lstg_format_name = 'ABIN')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c85a0e73/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 35dd73f..75c9c3e 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -283,7 +283,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
private String genExecFunc() {
// if the table to scan is not the fact table of cube, then it's a lookup table
- if (context.hasJoin == false && context.realization.getModel().isLookupTable(tableName)) {
+ if (context.realization.getModel().isLookupTable(tableName)) {
return "executeLookupTableQuery";
} else {
return "executeOLAPQuery";
[23/35] kylin git commit: KYLIN-2449 skip rewriting if no realiztion
in OLAPAggregateRel
Posted by li...@apache.org.
KYLIN-2449 skip rewriting if no realiztion in OLAPAggregateRel
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ddcc699
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ddcc699
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ddcc699
Branch: refs/heads/master-hbase0.98
Commit: 4ddcc699ec807aea73cf4c158550678803f48eef
Parents: 9250d9b
Author: etherge <et...@163.com>
Authored: Wed Feb 15 15:18:18 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Apr 2 08:56:34 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/ParameterDesc.java | 1 +
.../resources/query/sql_subquery/query16.sql | 38 ++++++++++++++++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 5 +--
3 files changed, 42 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 5ba2f14..272c4ee 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -34,6 +34,7 @@ import com.google.common.collect.Sets;
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class ParameterDesc implements Serializable {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/kylin-it/src/test/resources/query/sql_subquery/query16.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query16.sql b/kylin-it/src/test/resources/query/sql_subquery/query16.sql
new file mode 100644
index 0000000..5477e54
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query16.sql
@@ -0,0 +1,38 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ t1.leaf_categ_id, COUNT(*) AS nums
+FROM
+ (SELECT
+ f.leaf_categ_id
+ FROM
+ test_kylin_fact f inner join TEST_CATEGORY_GROUPINGS o on f.leaf_categ_id = o.leaf_categ_id and f.LSTG_SITE_ID = o.site_id
+ WHERE
+ f.lstg_format_name = 'ABIN') t1
+ INNER JOIN
+ (SELECT
+ leaf_categ_id
+ FROM
+ test_kylin_fact f
+ INNER JOIN test_order o ON f.order_id = o.order_id
+ WHERE
+ buyer_id > 100) t2 ON t1.leaf_categ_id = t2.leaf_categ_id
+GROUP BY t1.leaf_categ_id
+ORDER BY nums, leaf_categ_id
+limit 100
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ddcc699/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2c75a14..d9385e9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -279,7 +279,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
@Override
public void implementRewrite(RewriteImplementor implementor) {
// only rewrite the innermost aggregation
- if (!this.afterAggregate) {
+ boolean hasRealization = (null != this.context.realization);
+ if (hasRealization && !this.afterAggregate) {
translateAggregation();
buildRewriteFieldsAndMetricsColumns();
}
@@ -287,7 +288,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
implementor.visitChild(this, getInput());
// only rewrite the innermost aggregation
- if (!this.afterAggregate) {
+ if (hasRealization && !this.afterAggregate) {
// rewrite the aggCalls
this.rewriteAggCalls = new ArrayList<AggregateCall>(aggCalls.size());
for (int i = 0; i < this.aggCalls.size(); i++) {
[34/35] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b12173d..51b21b3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,13 +26,12 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -87,13 +86,14 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final BufferedMutator table;
+ final HTableInterface table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getBufferedMutator(htableName);
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getTable(htableName);
+ table.setAutoFlush(false, true);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.mutate(put);
+ table.put(put);
}
@Override
public void close() throws IOException {
- table.flush();
+ table.flushCommits();
table.close();
}
}
class Reader implements IGTScanner {
- final Table table;
+ final HTableInterface table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index e822ada..1875b1c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,9 +26,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -53,10 +52,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,7 +117,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -173,7 +172,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final AtomicReference<RuntimeException> regionErrorHolder = new AtomicReference<>();
try {
- Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
+ HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 951e2ef..b1059ae 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -25,12 +25,11 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -133,8 +132,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
+ HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index cde127e..1e550f9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -195,7 +195,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -234,7 +234,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = (HRegion)env.getRegion();
+ region = env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index feb4842..2814ad6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,8 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -80,8 +79,7 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin admin = conn.getAdmin();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -94,7 +92,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(TableName.valueOf(tableName))) {
+ if (admin.tableExists(tableName)) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -103,7 +101,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -112,7 +110,8 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -127,7 +126,8 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index df3cf08..eacff9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,21 +100,19 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
-
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 6587d4e..d5b36df 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final Table hTable;
+ private final HTableInterface hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,6 +117,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
+ hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 2f7e164..5b2441c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,20 +69,19 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Admin admin = null;
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin admin = null;
try {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- admin = conn.getAdmin();
-
+ admin = new HBaseAdmin(conf);
for (String table : oldTables) {
- if (admin.tableExists(TableName.valueOf(table))) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
+ if (admin.tableExists(table)) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(TableName.valueOf(table))) {
- admin.disableTable(TableName.valueOf(table));
+ if (admin.isTableEnabled(table)) {
+ admin.disableTable(table);
}
- admin.deleteTable(TableName.valueOf(table));
+ admin.deleteTable(table);
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index 56f867a..a150607 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,11 +21,9 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -40,8 +38,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -52,7 +50,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
+ hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 581de38..68c0a39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -26,19 +26,19 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -89,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static Admin hbaseAdmin;
+ private static HBaseAdmin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -134,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
hdfsFS = HadoopUtil.getWorkingFileSystem();
@@ -233,7 +233,6 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
-
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -327,8 +326,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -450,11 +449,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -474,6 +473,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -504,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- TableName tableName = TableName.valueOf((String) opt.params[0]);
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
+ String tableName = (String) opt.params[0];
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -539,12 +539,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +562,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config) {
+ private static void updateMeta(KylinConfig config){
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 20d0f7d..8bd4abf 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,8 +130,9 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ hbaseAdmin = new HBaseAdmin(conf);
+
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -188,10 +189,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.disableTable(sepNameList[0]);
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
- hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
+ hbaseAdmin.modifyTable(sepNameList[0], desc);
+ hbaseAdmin.enableTable(sepNameList[0]);
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 29c738e..c9e969f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,8 +44,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -86,8 +85,7 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -205,10 +203,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -229,7 +227,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -244,7 +242,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -270,15 +268,16 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.modifyTable(tableName, desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.enableTable(tableName);
return true;
}
- private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+
+ private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -299,12 +298,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final Admin hbaseAdmin;
+ private final HBaseAdmin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -425,7 +424,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 1cdb2f8..61c73d5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -236,9 +235,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -258,6 +257,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index dd5f8fa..86ba22f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(Connection conn) throws IOException {
- Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
+ private static void prepareData(HConnection conn) throws IOException {
+ HTableInterface table = conn.getTable(TEST_TABLE);
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 940d64a..6749d6c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,11 +24,9 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -57,8 +55,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -73,12 +71,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 1daca0a..937b65f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -32,15 +31,12 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionLocator;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -62,31 +58,30 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ public HBaseRegionSizeCalculator(HTable table) throws IOException {
+ this(table, new HBaseAdmin(table.getConfiguration()));
+ }
- Table table = null;
- Admin admin = null;
- try {
- table = hbaseConnection.getTable(TableName.valueOf(tableName));
- admin = hbaseConnection.getAdmin();
+ /** Constructor for unit testing */
+ HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ try {
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
+ logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
// Get regions for table.
- RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
- List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
+ Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionLocation hRegionLocation : regionLocationList) {
- tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
+ for (HRegionInfo regionInfo : tableRegionInfos) {
+ tableRegions.add(regionInfo.getRegionName());
}
- ClusterStatus clusterStatus = admin.getClusterStatus();
+ ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -110,7 +105,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(admin);
+ IOUtils.closeQuietly(hBaseAdmin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index a2f60d4..266f7e7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,10 +23,9 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -43,8 +42,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index 8dd2164..1db60fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- Connection conn = getConnection();
- Admin hadmin = conn.getAdmin();
+ HConnection conn = getConnection();
+ HBaseAdmin hadmin = new HBaseAdmin(conn);
try {
- boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
+ boolean tableExist = hadmin.tableExists(tableName);
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -120,8 +120,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
byte[] key = new byte[8 + 4];//time + id
@@ -136,7 +136,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.addColumn(CF, QN, cell);
+ put.add(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -172,8 +172,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- Connection conn = getConnection();
- Table table = conn.getTable(TableName.valueOf(tableName));
+ HConnection conn = getConnection();
+ HTableInterface table = conn.getTable(tableName);
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -208,7 +208,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(Table table) throws IOException {
+ private static long getFirstKeyTime(HTableInterface table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -226,8 +226,8 @@ public class HbaseStreamingInput {
}
- private static Connection getConnection() throws IOException {
- return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ private static HConnection getConnection() throws IOException {
+ return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ea05ab2..ca1a060 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,11 +23,10 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -51,8 +50,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index df4e912..8ff5b0f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,14 +30,10 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +52,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+
// get all kylin hbase tables
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -77,13 +73,12 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- TableName tableName = TableName.valueOf(htableName);
- if (hbaseAdmin.tableExists(tableName)) {
- if (hbaseAdmin.isTableEnabled(tableName)) {
- hbaseAdmin.disableTable(tableName);
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(tableName);
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index bba6745..1ea8e8d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,13 +22,12 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -60,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- Connection conn = null;
- Table table = null;
+ HConnection conn = null;
+ HTableInterface table = null;
ResultScanner scanner = null;
try {
- conn = ConnectionFactory.createConnection(hconf);
- table = conn.getTable(TableName.valueOf(hbaseTable));
+ conn = HConnectionManager.createConnection(hconf);
+ table = conn.getTable(hbaseTable);
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index db516bb..01edb1f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,12 +22,11 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -71,8 +70,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- Connection conn = ConnectionFactory.createConnection(conf);
- Table tableInterface = conn.getTable(TableName.valueOf(htableName));
+ HConnection conn = HConnectionManager.createConnection(conf);
+ HTableInterface tableInterface = conn.getTable(htableName);
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f6b65ab..23e7e10 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,9 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -59,7 +57,6 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,8 +77,7 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- Admin hbaseAdmin = conn.getAdmin();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -157,22 +153,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- Admin hbaseAdmin;
+ HBaseAdmin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
- if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
- hbaseAdmin.disableTable(TableName.valueOf(htableName));
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
}
- hbaseAdmin.deleteTable(TableName.valueOf(htableName));
+ hbaseAdmin.deleteTable(htableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index 42a54c8..e36f662 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,18 +24,16 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,15 +49,14 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private Admin hbaseAdmin;
+ private HBaseAdmin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
- hbaseAdmin = conn.getAdmin();
+ this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -169,9 +166,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(TableName.valueOf(tableName));
- hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
- hbaseAdmin.enableTable(TableName.valueOf(tableName));
+ hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.enableTable(tableName);
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 278c2b8..072ee44 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -60,16 +60,6 @@
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <scope>provided</scope>
- </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index c162a76..8ada323 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -462,11 +462,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table srcAclHtable = null;
- Table destAclHtable = null;
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -486,6 +486,7 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -551,12 +552,13 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- Table destAclHtable = null;
+ HTableInterface destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
+ destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index f52fc3e..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,11 +25,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -232,9 +231,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- Table aclHtable = null;
+ HTableInterface aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -254,6 +253,7 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
+ aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[33/35] kylin git commit: KYLIN-2165 fix IT
Posted by li...@apache.org.
KYLIN-2165 fix IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e87fb41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e87fb41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e87fb41
Branch: refs/heads/master-hbase0.98
Commit: 2e87fb41fe36aff015950bad3a88e7ac525debd6
Parents: 93cc5ab
Author: gaodayue <ga...@meituan.com>
Authored: Fri Apr 7 17:01:24 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Fri Apr 7 19:10:38 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/JoinedFlatTable.java | 32 +++----
.../kylin/job/engine/JobEngineConfig.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 4 +-
.../apache/kylin/source/hive/HiveMRInput.java | 95 ++++++++------------
4 files changed, 58 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e87fb41/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9ed563f..5553d34 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -20,12 +20,12 @@ package org.apache.kylin.job;
import java.io.File;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.JoinDesc;
@@ -46,30 +46,35 @@ public class JoinedFlatTable {
return storageDfsDir + "/" + flatDesc.getTableName();
}
- public static String generateHiveSetStatements(JobEngineConfig engineConfig) {
+ public static String generateHiveInitStatements(
+ String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) {
+
StringBuilder buffer = new StringBuilder();
+ buffer.append("USE ").append(flatTableDatabase).append(";\n");
try {
- File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath());
-
- if (hadoopPropertiesFile.exists()) {
+ File file = new File(kylinHiveFile);
+ if (file.exists()) {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
NodeList nl = doc.getElementsByTagName("property");
for (int i = 0; i < nl.getLength(); i++) {
String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue();
if (!name.equals("tmpjars")) {
- buffer.append("SET " + name + "=" + value + ";\n");
+ buffer.append("SET ").append(name).append("=").append(value).append(";\n");
}
}
}
} catch (Exception e) {
throw new RuntimeException("Failed to parse hive conf file ", e);
}
+
+ for (Map.Entry<String, String> entry : cubeOverrides.entrySet()) {
+ buffer.append("SET ").append(entry.getKey()).append("=").append(entry.getValue()).append(";\n");
+ }
+
return buffer.toString();
}
@@ -98,11 +103,8 @@ public class JoinedFlatTable {
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc, JobEngineConfig engineConfig) {
- StringBuilder sql = new StringBuilder();
- sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";").append("\n");
- return sql.toString();
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n";
}
public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e87fb41/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
index 0f5b7dd..8859527 100644
--- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
+++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java
@@ -95,7 +95,7 @@ public class JobEngineConfig {
return path;
}
- public String getHiveConfFilePath() throws IOException {
+ public String getHiveConfFilePath() {
String hiveConfFile = (HIVE_CONF_FILENAME + ".xml");
File jobConfig = getJobConfig(hiveConfFile);
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e87fb41/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
index 0faf22a..65169c9 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
@@ -22,13 +22,11 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.EngineFactory;
-import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.junit.After;
import org.junit.Before;
@@ -77,7 +75,7 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
@Test
public void testGenerateInsertSql() throws IOException {
- String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc);
System.out.println(sqls);
int length = sqls.length();
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e87fb41/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 2f966ab..418fcfc 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,7 +19,6 @@
package org.apache.kylin.source.hive;
import java.io.IOException;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -127,51 +126,58 @@ public class HiveMRInput implements IMRInput {
public static class BatchCubingInputSide implements IMRBatchCubingInputSide {
- JobEngineConfig conf;
final IJoinedFlatTableDesc flatDesc;
+ final String flatTableDatabase;
+ final String hdfsWorkingDir;
+
String hiveViewIntermediateTables = "";
public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
this.flatDesc = flatDesc;
+ this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
+ this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}
@Override
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
- final KylinConfig kylinConfig = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig();
- this.conf = new JobEngineConfig(kylinConfig);
+ final KylinConfig cubeConfig = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName).getConfig();
+ JobEngineConfig conf = new JobEngineConfig(cubeConfig);
+
+ final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(
+ flatTableDatabase, conf.getHiveConfFilePath(), cubeConfig.getHiveConfigOverride()
+ ) ;
+ final String jobWorkingDir = getJobWorkingDir(jobFlow);
// create flat table first, then count and redistribute
- jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName));
- if (kylinConfig.isHiveRedistributeEnabled() == true) {
- jobFlow.addTask(createRedistributeFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName));
+ jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, jobWorkingDir, cubeName));
+ if (cubeConfig.isHiveRedistributeEnabled() == true) {
+ jobFlow.addTask(createRedistributeFlatHiveTableStep(hiveInitStatements, cubeName));
}
- AbstractExecutable task = createLookupHiveViewMaterializationStep(jobFlow.getId());
+ AbstractExecutable task = createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir);
if (task != null) {
jobFlow.addTask(task);
}
}
- public static AbstractExecutable createRedistributeFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName) {
- StringBuilder hiveInitBuf = new StringBuilder();
- hiveInitBuf.append("USE ").append(conf.getConfig().getHiveDatabaseForIntermediateTable()).append(";\n");
- hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
- final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
- appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
+ private String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
+ return JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+ }
+ private AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName) {
RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
- step.setInitStatement(hiveInitBuf.toString());
- step.setIntermediateTable(flatTableDesc.getTableName());
- step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatTableDesc));
+ step.setInitStatement(hiveInitStatements);
+ step.setIntermediateTable(flatDesc.getTableName());
+ step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatDesc));
CubingExecutableUtil.setCubeName(cubeName, step.getParams());
step.setName(ExecutableConstants.STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE);
return step;
}
- public ShellExecutable createLookupHiveViewMaterializationStep(String jobId) {
+ private ShellExecutable createLookupHiveViewMaterializationStep(String hiveInitStatements, String jobWorkingDir) {
ShellExecutable step = new ShellExecutable();
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
- HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
MetadataManager metadataManager = MetadataManager.getInstance(kylinConfig);
@@ -187,16 +193,15 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
- appendHiveOverrideProperties2(kylinConfig, hiveCmdBuilder);
- final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";";
- hiveCmdBuilder.addStatement(useDatabaseHql);
- hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
+
+ HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(hiveInitStatements);
for (TableDesc lookUpTableDesc : lookupViewsTables) {
if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new StringBuilder();
createIntermediateTableHql.append("DROP TABLE IF EXISTS " + lookUpTableDesc.getMaterializedName() + ";\n");
createIntermediateTableHql.append("CREATE TABLE IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");
- createIntermediateTableHql.append("LOCATION '" + JobBuilderSupport.getJobWorkingDir(conf, jobId) + "/" + lookUpTableDesc.getMaterializedName() + "'\n");
+ createIntermediateTableHql.append("LOCATION '" + jobWorkingDir + "/" + lookUpTableDesc.getMaterializedName() + "'\n");
createIntermediateTableHql.append("AS SELECT * FROM " + lookUpTableDesc.getIdentity() + ";\n");
hiveCmdBuilder.addStatement(createIntermediateTableHql.toString());
hiveViewIntermediateTables = hiveViewIntermediateTables + lookUpTableDesc.getMaterializedName() + ";";
@@ -209,19 +214,14 @@ public class HiveMRInput implements IMRInput {
return step;
}
- public static AbstractExecutable createFlatHiveTableStep(JobEngineConfig conf, IJoinedFlatTableDesc flatTableDesc, String jobId, String cubeName) {
- StringBuilder hiveInitBuf = new StringBuilder();
- hiveInitBuf.append(JoinedFlatTable.generateHiveSetStatements(conf));
- final KylinConfig kylinConfig = ((CubeSegment) flatTableDesc.getSegment()).getConfig();
- appendHiveOverrideProperties(kylinConfig, hiveInitBuf);
- final String useDatabaseHql = "USE " + conf.getConfig().getHiveDatabaseForIntermediateTable() + ";\n";
- final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, JobBuilderSupport.getJobWorkingDir(conf, jobId));
- String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, conf);
+ private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir, String cubeName) {
+ final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
+ String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
- step.setInitStatement(hiveInitBuf.toString());
- step.setCreateTableStatement(useDatabaseHql + dropTableHql + createTableHql + insertDataHqls);
+ step.setInitStatement(hiveInitStatements);
+ step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
CubingExecutableUtil.setCubeName(cubeName, step.getParams());
step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
return step;
@@ -229,10 +229,12 @@ public class HiveMRInput implements IMRInput {
@Override
public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
+ final String jobWorkingDir = getJobWorkingDir(jobFlow);
+
GarbageCollectionStep step = new GarbageCollectionStep();
step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
step.setIntermediateTableIdentity(getIntermediateTableIdentity());
- step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, JobBuilderSupport.getJobWorkingDir(conf, jobFlow.getId())));
+ step.setExternalDataPath(JoinedFlatTable.getTableDir(flatDesc, jobWorkingDir));
step.setHiveViewIntermediateTableIdentities(hiveViewIntermediateTables);
jobFlow.addTask(step);
}
@@ -243,7 +245,7 @@ public class HiveMRInput implements IMRInput {
}
private String getIntermediateTableIdentity() {
- return conf.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatDesc.getTableName();
+ return flatTableDatabase + "." + flatDesc.getTableName();
}
}
@@ -413,24 +415,5 @@ public class HiveMRInput implements IMRInput {
public void setHiveViewIntermediateTableIdentities(String tableIdentities) {
setParam("oldHiveViewIntermediateTables", tableIdentities);
}
-
- }
-
- private static void appendHiveOverrideProperties(final KylinConfig kylinConfig, StringBuilder hiveCmd) {
- final Map<String, String> hiveConfOverride = kylinConfig.getHiveConfigOverride();
- if (hiveConfOverride.isEmpty() == false) {
- for (String key : hiveConfOverride.keySet()) {
- hiveCmd.append("SET ").append(key).append("=").append(hiveConfOverride.get(key)).append(";\n");
- }
- }
- }
-
- private static void appendHiveOverrideProperties2(final KylinConfig kylinConfig, HiveCmdBuilder hiveCmdBuilder) {
- final Map<String, String> hiveConfOverride = kylinConfig.getHiveConfigOverride();
- if (hiveConfOverride.isEmpty() == false) {
- for (String key : hiveConfOverride.keySet()) {
- hiveCmdBuilder.addStatement("SET " + key + "=" + hiveConfOverride.get(key) + ";\n");
- }
- }
}
}
[18/35] kylin git commit: KYLIN-2528 refine job email notification to
support starttls and customized port
Posted by li...@apache.org.
KYLIN-2528 refine job email notification to support starttls and customized port
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6ee76c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6ee76c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6ee76c8
Branch: refs/heads/master-hbase0.98
Commit: f6ee76c85a86971e71f998b2dd096b035bc7613c
Parents: 19d5b3d
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 31 18:12:22 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 18:12:22 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 12 +++++++-----
.../org/apache/kylin/common/KylinConfigBase.java | 8 ++++++++
.../org/apache/kylin/common/util/MailService.java | 15 +++++++++++++--
examples/test_case_data/localmeta/kylin.properties | 15 +++++++++------
examples/test_case_data/sandbox/kylin.properties | 8 +++++++-
5 files changed, 44 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1a55c94..0a77093 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -117,11 +117,13 @@ kylin.job.status.with.kerberos=false
kylin.job.step.timeout=7200
# If true, will send email notification on job complete
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=kylin@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=kylin@example.com
### ENGINE ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 02349ad..36c2b16 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -392,6 +392,14 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false"));
}
+ public boolean isStarttlsEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.job.notification-mail-enable-starttls", "false"));
+ }
+
+ public String getSmtpPort() {
+ return getOptional("kylin.job.notification-mail-port", "25");
+ }
+
public String getMailHost() {
return getOptional("kylin.job.notification-mail-host", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 5793967..25bc03c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -34,18 +34,22 @@ public class MailService {
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(MailService.class);
private Boolean enabled = Boolean.TRUE;
+ private Boolean starttlsEnabled = Boolean.FALSE;
private String host;
+ private String port;
private String username;
private String password;
private String sender;
public MailService(KylinConfig config) {
- this(config.isMailEnabled(), config.getMailHost(), config.getMailUsername(), config.getMailPassword(), config.getMailSender());
+ this(config.isMailEnabled(), config.isStarttlsEnabled(), config.getMailHost(), config.getSmtpPort(), config.getMailUsername(), config.getMailPassword(), config.getMailSender());
}
- private MailService(boolean enabled, String host, String username, String password, String sender) {
+ private MailService(boolean enabled, boolean starttlsEnabled, String host, String port, String username, String password, String sender) {
this.enabled = enabled;
+ this.starttlsEnabled = starttlsEnabled;
this.host = host;
+ this.port = port;
this.username = username;
this.password = password;
this.sender = sender;
@@ -85,6 +89,13 @@ public class MailService {
Email email = new HtmlEmail();
email.setHostName(host);
+ email.setStartTLSEnabled(starttlsEnabled);
+ if (starttlsEnabled) {
+ email.setSslSmtpPort(port);
+ } else {
+ email.setSmtpPort(Integer.valueOf(port));
+ }
+
if (username != null && username.trim().length() > 0) {
email.setAuthentication(username, password);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index eb4f35d..9f7b24c 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -118,11 +118,14 @@ kylin.security.saml.context-path=/kylin
### MAIL ###
# If true, will send email notification;
-kylin.job.notification-enabled=false
-kylin.job.notification-mail-host=mail.com
-kylin.job.notification-mail-username=
-kylin.job.notification-mail-password=need_reset
-kylin.job.notification-mail-sender=
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=kylin@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=kylin@example.com
+
### OTHER ###
@@ -133,4 +136,4 @@ kylin.engine.mr.config-override.test2=test2
kylin.job.lock=org.apache.kylin.job.lock.MockJobLock
kylin.engine.provider.0=org.apache.kylin.engine.mr.MRBatchCubingEngine
-kylin.engine.provider.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
\ No newline at end of file
+kylin.engine.provider.2=org.apache.kylin.engine.mr.MRBatchCubingEngine2
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee76c8/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 6e360a3..1674d82 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -139,7 +139,13 @@ kylin.security.acl.admin-role=ROLE_ADMIN
### MAIL ###
# If true, will send email notification;
-kylin.job.notification-enabled=false
+#kylin.job.notification-enabled=true
+#kylin.job.notification-mail-enable-starttls=true
+#kylin.job.notification-mail-host=smtp.office365.com
+#kylin.job.notification-mail-port=587
+#kylin.job.notification-mail-username=kylin@example.com
+#kylin.job.notification-mail-password=mypassword
+#kylin.job.notification-mail-sender=kylin@example.com
### WEB ###
[03/35] kylin git commit: Merge commit
'88a1c71dde855c693b230f67b92c4cd067d43b2b'
Posted by li...@apache.org.
Merge commit '88a1c71dde855c693b230f67b92c4cd067d43b2b'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1cc0dd6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1cc0dd6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1cc0dd6
Branch: refs/heads/master-hbase0.98
Commit: b1cc0dd643ebd55776a3f22b61ce33e2abf29d48
Parents: 636282d 88a1c71
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Mar 27 15:39:03 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 27 15:39:03 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/ParamAsMeasureCount.java | 30 ++++++++++++++++++++
.../resources/query/sql_distinct/query08.sql | 24 ++++++++++++++++
2 files changed, 54 insertions(+)
----------------------------------------------------------------------
[29/35] kylin git commit: KYLIN-2501 pipeline partition results if
possible
Posted by li...@apache.org.
KYLIN-2501 pipeline partition results if possible
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bce27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bce27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bce27
Branch: refs/heads/master-hbase0.98
Commit: 6f9bce27545ef185a64c24aff67790150fb469b9
Parents: 7cb88f5
Author: gaodayue <ga...@meituan.com>
Authored: Wed Apr 5 13:47:34 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Wed Apr 5 13:56:03 2017 +0800
----------------------------------------------------------------------
.../gtrecord/StorageResponseGTScatter.java | 22 +++++++++++---------
1 file changed, 12 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bce27/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index f1ab20c..ef12ff0 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.gtrecord;
+import com.google.common.base.Function;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -69,21 +70,22 @@ public class StorageResponseGTScatter implements IGTScanner {
@Override
public Iterator<GTRecord> iterator() {
- List<PartitionResultIterator> partitionResults = Lists.newArrayList();
- while (blocks.hasNext()) {
- partitionResults.add(new PartitionResultIterator(blocks.next(), info, columns));
+ Iterator<PartitionResultIterator> iterators = Iterators.transform(blocks, new Function<byte[], PartitionResultIterator>() {
+ public PartitionResultIterator apply(byte[] input) {
+ return new PartitionResultIterator(input, info, columns);
+ }
+ });
+
+ if (!needSorted) {
+ logger.debug("Using Iterators.concat to pipeline partition results");
+ return Iterators.concat(iterators);
}
+ List<PartitionResultIterator> partitionResults = Lists.newArrayList(iterators);
if (partitionResults.size() == 1) {
return partitionResults.get(0);
}
-
- if (!needSorted) {
- logger.debug("Using Iterators.concat to merge partition results");
- return Iterators.concat(partitionResults.iterator());
- }
-
- logger.debug("Using SortMergedPartitionResultIterator to merge partition results");
+ logger.debug("Using SortMergedPartitionResultIterator to merge {} partition results", partitionResults.size());
return new SortMergedPartitionResultIterator(partitionResults, info, GTRecord.getComparator(groupByDims));
}
}
[28/35] kylin git commit: KYLIN-2530 pass hive configuration along
with MR job
Posted by li...@apache.org.
KYLIN-2530 pass hive configuration along with MR job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7cb88f5f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7cb88f5f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7cb88f5f
Branch: refs/heads/master-hbase0.98
Commit: 7cb88f5f971535cdd43292c624c3a3f08faadcdf
Parents: 551f5ed
Author: Yang Li <li...@apache.org>
Authored: Tue Apr 4 16:55:53 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Apr 4 16:56:12 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7cb88f5f/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index bbf3c60..a5d1353 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -107,6 +107,8 @@ public class HiveMRInput implements IMRInput {
@Override
public void configureJob(Job job) {
try {
+ job.getConfiguration().addResource("hive-site.xml");
+
HCatInputFormat.setInput(job, dbName, tableName);
job.setInputFormatClass(HCatInputFormat.class);
[31/35] kylin git commit: KYLIN-2165 use cube specific configs for
hive job and use lowercase tablename
Posted by li...@apache.org.
KYLIN-2165 use cube specific configs for hive job and use lowercase tablename
Signed-off-by: gaodayue <ga...@meituan.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7129ea65
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7129ea65
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7129ea65
Branch: refs/heads/master-hbase0.98
Commit: 7129ea650e757d6c50a81823760c45734688cfd2
Parents: 4ae40bd
Author: gaodayue <ga...@meituan.com>
Authored: Thu Apr 6 13:55:27 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Thu Apr 6 20:38:03 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java | 4 ++--
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7129ea65/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 2a68116..f607466 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -63,9 +63,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Se
protected String makeTableName(CubeDesc cubeDesc, CubeSegment cubeSegment) {
if (cubeSegment == null) {
- return "kylin_intermediate_" + cubeDesc.getName();
+ return "kylin_intermediate_" + cubeDesc.getName().toLowerCase();
} else {
- return "kylin_intermediate_" + cubeDesc.getName() + "_" + cubeSegment.getUuid().replaceAll("-", "_");
+ return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + "_" + cubeSegment.getUuid().replaceAll("-", "_");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7129ea65/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index a5d1353..0a03461 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -127,12 +127,11 @@ public class HiveMRInput implements IMRInput {
public static class BatchCubingInputSide implements IMRBatchCubingInputSide {
- final JobEngineConfig conf;
+ JobEngineConfig conf;
final IJoinedFlatTableDesc flatDesc;
String hiveViewIntermediateTables = "";
public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
- this.conf = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
this.flatDesc = flatDesc;
}
@@ -140,6 +139,7 @@ public class HiveMRInput implements IMRInput {
public void addStepPhase1_CreateFlatTable(DefaultChainedExecutable jobFlow) {
final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
final KylinConfig kylinConfig = CubeManager.getInstance(conf.getConfig()).getCube(cubeName).getConfig();
+ this.conf = new JobEngineConfig(kylinConfig);
// create flat table first, then count and redistribute
jobFlow.addTask(createFlatHiveTableStep(conf, flatDesc, jobFlow.getId(), cubeName));
[25/35] kylin git commit: KYLIN-2341 support sum(case.. when..)
Posted by li...@apache.org.
KYLIN-2341 support sum(case.. when..)
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/20578319
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/20578319
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/20578319
Branch: refs/heads/master-hbase0.98
Commit: 20578319e11e504c2ba647a0593dd6eae2d4624b
Parents: c85a0e7
Author: etherge <et...@163.com>
Authored: Sun Feb 19 16:49:08 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Apr 3 16:19:46 2017 +0800
----------------------------------------------------------------------
.../resources/query/sql_casewhen/query04.sql | 30 ++++++++++++++++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +--
.../kylin/query/relnode/OLAPProjectRel.java | 10 +------
3 files changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/20578319/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_casewhen/query04.sql b/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
new file mode 100644
index 0000000..646da0a
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
@@ -0,0 +1,30 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ SUM(CASE
+ WHEN lstg_format_name LIKE 'Other%' THEN price
+ ELSE 0
+ END) AS gmv
+FROM
+ test_kylin_fact
+ INNER JOIN
+ edw.test_cal_dt AS test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ INNER JOIN
+ test_category_groupings ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
+ AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/20578319/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index d9385e9..c7f66a0 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -209,12 +209,12 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
for (int i = 0; i < this.aggregations.size(); i++) {
FunctionDesc aggFunc = this.aggregations.get(i);
String aggOutName;
- if (aggFunc.needRewriteField()) {
+ if (aggFunc != null && aggFunc.needRewriteField()) {
aggOutName = aggFunc.getRewriteFieldName();
} else {
AggregateCall aggCall = this.rewriteAggCalls.get(i);
int index = aggCall.getArgList().get(0);
- aggOutName = aggFunc.getExpression() + "_" + inputColumnRowType.getColumnByIndex(index).getIdentity() + "_";
+ aggOutName = getSqlFuncName(aggCall) + "_" + inputColumnRowType.getColumnByIndex(index).getIdentity() + "_";
}
TblColRef aggOutCol = TblColRef.newInnerColumn(aggOutName, TblColRef.InnerDataTypeEnum.LITERAL);
columns.add(aggOutCol);
http://git-wip-us.apache.org/repos/asf/kylin/blob/20578319/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 03b9ddd..aa03e3e 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -37,9 +37,9 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
-import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -47,7 +47,6 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexProgram;
import org.apache.calcite.sql.SqlOperator;
-import org.apache.calcite.sql.fun.SqlCaseOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
import org.apache.kylin.metadata.model.TblColRef;
@@ -200,13 +199,6 @@ public class OLAPProjectRel extends Project implements OLAPRel {
if (operator.getName().equals("QUARTER")) {
return translateFirstRexInputRef(call, inputColumnRowType, fieldName, sourceCollector);
}
- } else if (operator instanceof SqlCaseOperator) {
- for (RexNode operand : call.getOperands()) {
- if (operand instanceof RexInputRef) {
- RexInputRef inputRef = (RexInputRef) operand;
- return translateRexInputRef(inputRef, inputColumnRowType, fieldName, sourceCollector);
- }
- }
}
for (RexNode operand : call.getOperands()) {
[26/35] kylin git commit: KYLIN-2341 report error on unsupported aggr
func
Posted by li...@apache.org.
KYLIN-2341 report error on unsupported aggr func
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7af5529e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7af5529e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7af5529e
Branch: refs/heads/master-hbase0.98
Commit: 7af5529ef566434b34131a6b77c0f32e738bb318
Parents: 2057831
Author: Yang Li <li...@apache.org>
Authored: Mon Apr 3 22:35:21 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Apr 3 22:35:21 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/FunctionDesc.java | 8 +--
.../kylin/metadata/model/ParameterDesc.java | 57 ++++++++++----------
.../apache/kylin/metadata/model/TblColRef.java | 10 ++++
.../resources/query/sql_casewhen/query04.sql | 30 -----------
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../kylin/query/relnode/OLAPProjectRel.java | 2 +-
6 files changed, 44 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 61c5fac..e969be2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,7 +19,6 @@
package org.apache.kylin.metadata.model;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
@@ -41,6 +40,7 @@ import com.google.common.collect.Sets;
/**
*/
+@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class FunctionDesc implements Serializable {
@@ -90,17 +90,13 @@ public class FunctionDesc implements Serializable {
expression = expression.toUpperCase();
returnDataType = DataType.getType(returnType);
- ArrayList<TblColRef> colRefs = Lists.newArrayList();
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
p.setValue(colRef.getIdentity());
- colRefs.add(colRef);
+ p.setColRef(colRef);
}
}
-
- if (parameter != null)
- parameter.setColRefs(colRefs);
}
private void reInitMeasureType() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 272c4ee..0cf31b1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
@@ -28,8 +29,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
/**
@@ -49,7 +48,7 @@ public class ParameterDesc implements Serializable {
TblColRef col = (TblColRef) obj;
r.type = FunctionDesc.PARAMETER_TYPE_COLUMN;
r.value = col.getIdentity();
- r.colRefs = ImmutableList.of(col);
+ r.colRef = col;
} else {
r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT;
r.value = (String) obj;
@@ -57,12 +56,6 @@ public class ParameterDesc implements Serializable {
if (objs.length >= 2) {
r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1, objs.length));
- if (r.nextParameter.colRefs.size() > 0) {
- if (r.colRefs.isEmpty())
- r.colRefs = r.nextParameter.colRefs;
- else
- r.colRefs = ImmutableList.copyOf(Iterables.concat(r.colRefs, r.nextParameter.colRefs));
- }
}
return r;
}
@@ -76,7 +69,8 @@ public class ParameterDesc implements Serializable {
@JsonInclude(JsonInclude.Include.NON_NULL)
private ParameterDesc nextParameter;
- private List<TblColRef> colRefs = ImmutableList.of();
+ private TblColRef colRef = null;
+ private List<TblColRef> allColRefsIncludingNexts = null;
private Set<PlainParameter> plainParameters = null;
// Lazy evaluation
@@ -102,13 +96,28 @@ public class ParameterDesc implements Serializable {
void setValue(String value) {
this.value = value;
}
+
+ public TblColRef getColRef() {
+ return colRef;
+ }
- public List<TblColRef> getColRefs() {
- return colRefs;
+ void setColRef(TblColRef colRef) {
+ this.colRef = colRef;
}
- void setColRefs(List<TblColRef> colRefs) {
- this.colRefs = colRefs;
+ public List<TblColRef> getColRefs() {
+ if (allColRefsIncludingNexts == null) {
+ List<TblColRef> all = new ArrayList<>(2);
+ ParameterDesc p = this;
+ while (p != null) {
+ if (p.isColumnType())
+ all.add(p.getColRef());
+
+ p = p.nextParameter;
+ }
+ allColRefsIncludingNexts = all;
+ }
+ return allColRefsIncludingNexts;
}
public ParameterDesc getNextParameter() {
@@ -132,17 +141,12 @@ public class ParameterDesc implements Serializable {
return false;
ParameterDesc p = this, q = that;
- int refi = 0, refj = 0;
for (; p != null && q != null; p = p.nextParameter, q = q.nextParameter) {
if (p.isColumnType()) {
if (q.isColumnType() == false)
return false;
- if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
- return false;
- if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == false)
+ if (this.getColRef().equals(that.getColRef()) == false)
return false;
- refi++;
- refj++;
} else {
if (q.isColumnType() == true)
return false;
@@ -171,13 +175,14 @@ public class ParameterDesc implements Serializable {
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
- result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
+ result = 31 * result + (colRef != null ? colRef.hashCode() : 0);
return result;
}
@Override
public String toString() {
- return "ParameterDesc [type=" + type + ", value=" + value + ", nextParam=" + nextParameter + "]";
+ String thisStr = isColumnType() ? colRef.toString() : value;
+ return nextParameter == null ? thisStr : thisStr + "," + nextParameter.toString();
}
/**
@@ -201,11 +206,9 @@ public class ParameterDesc implements Serializable {
static Set<PlainParameter> createFromParameterDesc(ParameterDesc parameterDesc) {
Set<PlainParameter> result = Sets.newHashSet();
ParameterDesc local = parameterDesc;
- List<TblColRef> totalColRef = parameterDesc.colRefs;
- Integer colIndex = 0;
while (local != null) {
if (local.isColumnType()) {
- result.add(createSingleColumnParameter(local, totalColRef.get(colIndex++)));
+ result.add(createSingleColumnParameter(local));
} else {
result.add(createSingleValueParameter(local));
}
@@ -221,11 +224,11 @@ public class ParameterDesc implements Serializable {
return single;
}
- static PlainParameter createSingleColumnParameter(ParameterDesc parameterDesc, TblColRef colRef) {
+ static PlainParameter createSingleColumnParameter(ParameterDesc parameterDesc) {
PlainParameter single = new PlainParameter();
single.type = parameterDesc.type;
single.value = parameterDesc.value;
- single.colRef = colRef;
+ single.colRef = parameterDesc.colRef;
return single;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 88abb9d..4661bc9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -54,12 +54,18 @@ public class TblColRef implements Serializable {
// used by projection rewrite, see OLAPProjectRel
public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType) {
+ return newInnerColumn(columnName, dataType, null);
+ }
+
+ // used by projection rewrite, see OLAPProjectRel
+ public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType, String parserDescription) {
ColumnDesc column = new ColumnDesc();
column.setName(columnName);
TableDesc table = new TableDesc();
column.setTable(table);
TblColRef colRef = new TblColRef(column);
colRef.markInnerColumn(dataType);
+ colRef.parserDescription = parserDescription;
return colRef;
}
@@ -101,6 +107,7 @@ public class TblColRef implements Serializable {
private TableRef table;
private ColumnDesc column;
private String identity;
+ private String parserDescription;
TblColRef(ColumnDesc column) {
this.column = column;
@@ -198,6 +205,9 @@ public class TblColRef implements Serializable {
@Override
public String toString() {
+ if (isInnerColumn() && parserDescription != null)
+ return parserDescription;
+
String alias = table == null ? "UNKNOWN_MODEL" : table.getAlias();
String tableName = column.getTable() == null ? "NULL" : column.getTable().getName();
String tableIdentity = column.getTable() == null ? "NULL" : column.getTable().getIdentity();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_casewhen/query04.sql b/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
deleted file mode 100644
index 646da0a..0000000
--- a/kylin-it/src/test/resources/query/sql_casewhen/query04.sql
+++ /dev/null
@@ -1,30 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT
- SUM(CASE
- WHEN lstg_format_name LIKE 'Other%' THEN price
- ELSE 0
- END) AS gmv
-FROM
- test_kylin_fact
- INNER JOIN
- edw.test_cal_dt AS test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- INNER JOIN
- test_category_groupings ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id
- AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index c7f66a0..b669e3b 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -262,9 +262,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
for (Integer index : aggCall.getArgList().subList(0, columnsCount)) {
TblColRef column = inputColumnRowType.getColumnByIndex(index);
- if (!column.isInnerColumn()) {
- columns.add(column);
- }
+ columns.add(column);
}
if (!columns.isEmpty()) {
parameter = ParameterDesc.newInstance(columns.toArray(new TblColRef[columns.size()]));
http://git-wip-us.apache.org/repos/asf/kylin/blob/7af5529e/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index aa03e3e..1d0de40 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -204,7 +204,7 @@ public class OLAPProjectRel extends Project implements OLAPRel {
for (RexNode operand : call.getOperands()) {
translateRexNode(operand, inputColumnRowType, fieldName, sourceCollector);
}
- return TblColRef.newInnerColumn(fieldName, InnerDataTypeEnum.LITERAL);
+ return TblColRef.newInnerColumn(fieldName, InnerDataTypeEnum.LITERAL, call.toString());
}
@Override
[30/35] kylin git commit: minor,
recover query test sql_casewhen/query55.sql
Posted by li...@apache.org.
minor, recover query test sql_casewhen/query55.sql
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ae40bd7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ae40bd7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ae40bd7
Branch: refs/heads/master-hbase0.98
Commit: 4ae40bd7a59217adef7fe68f952b006191908551
Parents: 6f9bce2
Author: Li Yang <li...@apache.org>
Authored: Thu Apr 6 15:46:43 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Apr 6 15:46:43 2017 +0800
----------------------------------------------------------------------
.../resources/query/sql_casewhen/query55.sql | 22 ++++++++++++++++++++
.../query/sql_casewhen/query55.sql.disable | 22 --------------------
2 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae40bd7/kylin-it/src/test/resources/query/sql_casewhen/query55.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_casewhen/query55.sql b/kylin-it/src/test/resources/query/sql_casewhen/query55.sql
new file mode 100644
index 0000000..d29b60d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_casewhen/query55.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END) AS "LSTG_FORMAT_NAME__group_",
+ SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
+FROM "TEST_KYLIN_FACT" "TEST_KYLIN_FACT"
+GROUP BY (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END)
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae40bd7/kylin-it/src/test/resources/query/sql_casewhen/query55.sql.disable
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_casewhen/query55.sql.disable b/kylin-it/src/test/resources/query/sql_casewhen/query55.sql.disable
deleted file mode 100644
index d29b60d..0000000
--- a/kylin-it/src/test/resources/query/sql_casewhen/query55.sql.disable
+++ /dev/null
@@ -1,22 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END) AS "LSTG_FORMAT_NAME__group_",
- SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
-FROM "TEST_KYLIN_FACT" "TEST_KYLIN_FACT"
-GROUP BY (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END)
[12/35] kylin git commit: KYLIN-2501 bugfix, pass IT
Posted by li...@apache.org.
KYLIN-2501 bugfix, pass IT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/782a9748
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/782a9748
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/782a9748
Branch: refs/heads/master-hbase0.98
Commit: 782a97482ca1e92cf24e04badd8ee48c9b829f46
Parents: 0fa5724
Author: gaodayue <ga...@meituan.com>
Authored: Wed Mar 22 16:31:45 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 16:39:35 2017 +0800
----------------------------------------------------------------------
.../gridtable/GTStreamAggregateScanner.java | 24 +++--
.../apache/kylin/storage/StorageContext.java | 12 ---
.../gtrecord/GTCubeStorageQueryBase.java | 7 --
.../storage/gtrecord/PartitionResultMerger.java | 100 -------------------
.../gtrecord/SegmentCubeTupleIterator.java | 7 +-
.../SortMergedPartitionResultIterator.java | 81 +++++++++++++++
.../gtrecord/StorageResponseGTScatter.java | 13 ++-
7 files changed, 108 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
index 1fde423..4eb5791 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTStreamAggregateScanner.java
@@ -18,6 +18,7 @@
package org.apache.kylin.gridtable;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import org.apache.kylin.GTForwardingScanner;
@@ -38,11 +39,10 @@ public class GTStreamAggregateScanner extends GTForwardingScanner {
private final GTScanRequest req;
private final Comparator<GTRecord> keyComparator;
- public GTStreamAggregateScanner(IGTScanner delegated,
- GTScanRequest req, Comparator<GTRecord> keyComparator) {
+ public GTStreamAggregateScanner(IGTScanner delegated, GTScanRequest scanRequest) {
super(delegated);
- this.req = req;
- this.keyComparator = keyComparator;
+ this.req = Preconditions.checkNotNull(scanRequest, "scanRequest");
+ this.keyComparator = GTRecord.getComparator(scanRequest.getAggrGroupBy());
}
@Override
@@ -172,14 +172,22 @@ public class GTStreamAggregateScanner extends GTForwardingScanner {
private class StreamMergeValuesIterator extends AbstractStreamMergeIterator<Object[]> {
private int[] gtDimsIdx;
- private int[] gtMetricsIdx;
+ private int[] gtMetricsIdx; // specify which metric to return and their order
+ private int[] aggIdx; // specify the ith returning metric's aggStates index
+
private Object[] result; // avoid object creation
StreamMergeValuesIterator(Iterator<GTRecord> input, int[] gtDimsIdx, int[] gtMetricsIdx) {
super(input);
this.gtDimsIdx = gtDimsIdx;
this.gtMetricsIdx = gtMetricsIdx;
- result = new Object[gtDimsIdx.length + gtMetricsIdx.length];
+ this.aggIdx = new int[gtMetricsIdx.length];
+ for (int i = 0; i < aggIdx.length; i++) {
+ int metricIdx = gtMetricsIdx[i];
+ aggIdx[i] = metrics.trueBitIndexOf(metricIdx);
+ }
+
+ this.result = new Object[gtDimsIdx.length + gtMetricsIdx.length];
}
private void decodeAndSetDimensions(GTRecord record) {
@@ -202,8 +210,8 @@ public class GTStreamAggregateScanner extends GTForwardingScanner {
protected Object[] finalizeResult(GTRecord record, Object[] aggStates) {
decodeAndSetDimensions(record);
// set metrics
- for (int i = 0; i < gtMetricsIdx.length; i++) {
- result[gtDimsIdx.length + i] = aggStates[i];
+ for (int i = 0; i < aggIdx.length; i++) {
+ result[gtDimsIdx.length + i] = aggStates[aggIdx[i]];
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index bb17054..4522261 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -18,12 +18,10 @@
package org.apache.kylin.storage;
-import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.storage.gtrecord.GTCubeStorageQueryBase;
import org.slf4j.Logger;
@@ -49,9 +47,7 @@ public class StorageContext {
private boolean exactAggregation = false;
private boolean needStorageAggregation = false;
private boolean enableCoprocessor = false;
-
private boolean enableStreamAggregate = false;
- private Comparator<GTRecord> groupKeyComparator;
private IStorageQuery storageQuery;
private AtomicLong processedRowCount = new AtomicLong();
@@ -242,12 +238,4 @@ public class StorageContext {
public void enableStreamAggregate() {
this.enableStreamAggregate = true;
}
-
- public Comparator<GTRecord> getGroupKeyComparator() {
- return groupKeyComparator;
- }
-
- public void setGroupKeyComparator(Comparator<GTRecord> groupKeyComparator) {
- this.groupKeyComparator = groupKeyComparator;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 82590a2..d91a0b4 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -26,18 +26,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -392,11 +389,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
if (enabled) {
- CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
- ImmutableBitSet cols = mapping.makeGridTableColumns(groupsD);
-
context.enableStreamAggregate();
- context.setGroupKeyComparator(GTRecord.getComparator(cols));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
deleted file mode 100644
index 52029d3..0000000
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/PartitionResultMerger.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.gtrecord;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-import com.google.common.collect.UnmodifiableIterator;
-import org.apache.kylin.gridtable.GTInfo;
-import org.apache.kylin.gridtable.GTRecord;
-
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.PriorityQueue;
-
-/**
- * Merge-sort {@code GTRecord}s in all partitions, assume each partition contains sorted elements.
- */
-public class PartitionResultMerger implements Iterable<GTRecord> {
- private final ImmutableList<PartitionResultIterator> partitionResults;
- private final GTInfo info;
- private final Comparator<GTRecord> comparator;
-
- public PartitionResultMerger(
- Iterable<PartitionResultIterator> partitionResults,
- GTInfo info, Comparator<GTRecord> comparator) {
- this.partitionResults = ImmutableList.copyOf(partitionResults);
- this.info = info;
- this.comparator = comparator;
- }
-
- @Override
- public Iterator<GTRecord> iterator() {
- if (partitionResults.size() == 1) {
- return partitionResults.get(0);
- }
- return new MergingResultsIterator();
- }
-
- private class MergingResultsIterator extends UnmodifiableIterator<GTRecord> {
- final GTRecord record = new GTRecord(info); // reuse to avoid object creation
-
- PriorityQueue<PeekingIterator<GTRecord>> heap;
-
- MergingResultsIterator() {
- Comparator<PeekingIterator<GTRecord>> heapComparator = new Comparator<PeekingIterator<GTRecord>>() {
- public int compare(PeekingIterator<GTRecord> o1, PeekingIterator<GTRecord> o2) {
- return comparator.compare(o1.peek(), o2.peek());
- }
- };
- this.heap = new PriorityQueue<>(partitionResults.size(), heapComparator);
-
- for (PartitionResultIterator it : partitionResults) {
- if (it.hasNext()) {
- heap.offer(Iterators.peekingIterator(it));
- }
- }
- }
-
- @Override
- public boolean hasNext() {
- return !heap.isEmpty();
- }
-
- @Override
- public GTRecord next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- // get smallest record
- PeekingIterator<GTRecord> it = heap.poll();
- // WATCH OUT! record got from PartitionResultIterator.next() may changed later,
- // so we must make a shallow copy of it.
- record.shallowCopyFrom(it.next());
-
- if (it.hasNext()) {
- heap.offer(it);
- }
-
- return record;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
index 11f766c..3bac5ec 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SegmentCubeTupleIterator.java
@@ -90,8 +90,8 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
final Iterator<GTRecord> records, final GTScanRequest scanRequest,
final int[] gtDimsIdx, final int[] gtMetricsIdx) {
- boolean singlePartitionResult = records instanceof PartitionResultIterator;
- if (context.isStreamAggregateEnabled() && !singlePartitionResult) {
+ boolean hasMultiplePartitions = records instanceof SortMergedPartitionResultIterator;
+ if (hasMultiplePartitions && context.isStreamAggregateEnabled()) {
// input records are ordered, leverage stream aggregator to produce possibly fewer records
IGTScanner inputScanner = new IGTScanner() {
public GTInfo getInfo() {
@@ -104,8 +104,7 @@ public class SegmentCubeTupleIterator implements ITupleIterator {
return records;
}
};
- GTStreamAggregateScanner aggregator = new GTStreamAggregateScanner(
- inputScanner, scanRequest, context.getGroupKeyComparator());
+ GTStreamAggregateScanner aggregator = new GTStreamAggregateScanner(inputScanner, scanRequest);
return aggregator.valuesIterator(gtDimsIdx, gtMetricsIdx);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java
new file mode 100644
index 0000000..21e61e3
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SortMergedPartitionResultIterator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import com.google.common.collect.UnmodifiableIterator;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.PriorityQueue;
+
+/**
+ * Merge-sort {@code GTRecord}s in all partitions, assume each partition contains sorted elements.
+ */
+public class SortMergedPartitionResultIterator extends UnmodifiableIterator<GTRecord> {
+
+ final GTRecord record ; // reuse to avoid object creation
+ PriorityQueue<PeekingIterator<GTRecord>> heap;
+
+ SortMergedPartitionResultIterator(
+ List<PartitionResultIterator> partitionResults,
+ GTInfo info, final Comparator<GTRecord> comparator) {
+
+ this.record = new GTRecord(info);
+ Comparator<PeekingIterator<GTRecord>> heapComparator = new Comparator<PeekingIterator<GTRecord>>() {
+ public int compare(PeekingIterator<GTRecord> o1, PeekingIterator<GTRecord> o2) {
+ return comparator.compare(o1.peek(), o2.peek());
+ }
+ };
+ this.heap = new PriorityQueue<>(partitionResults.size(), heapComparator);
+
+ for (PartitionResultIterator it : partitionResults) {
+ if (it.hasNext()) {
+ heap.offer(Iterators.peekingIterator(it));
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !heap.isEmpty();
+ }
+
+ @Override
+ public GTRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ // get smallest record
+ PeekingIterator<GTRecord> it = heap.poll();
+ // WATCH OUT! record got from PartitionResultIterator.next() may changed later,
+ // so we must make a shallow copy of it.
+ record.shallowCopyFrom(it.next());
+
+ if (it.hasNext()) {
+ heap.offer(it);
+ }
+
+ return record;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/782a9748/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 0f1e191..f1ab20c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -44,7 +44,7 @@ public class StorageResponseGTScatter implements IGTScanner {
private IPartitionStreamer partitionStreamer;
private final Iterator<byte[]> blocks;
private final ImmutableBitSet columns;
- private final StorageContext context;
+ private final ImmutableBitSet groupByDims;
private final boolean needSorted; // whether scanner should return sorted records
public StorageResponseGTScatter(GTScanRequest scanRequest, IPartitionStreamer partitionStreamer, StorageContext context) {
@@ -52,7 +52,7 @@ public class StorageResponseGTScatter implements IGTScanner {
this.partitionStreamer = partitionStreamer;
this.blocks = partitionStreamer.asByteArrayIterator();
this.columns = scanRequest.getColumns();
- this.context = context;
+ this.groupByDims = scanRequest.getAggrGroupBy();
this.needSorted = (context.getFinalPushDownLimit() != Integer.MAX_VALUE) || context.isStreamAggregateEnabled();
}
@@ -74,13 +74,16 @@ public class StorageResponseGTScatter implements IGTScanner {
partitionResults.add(new PartitionResultIterator(blocks.next(), info, columns));
}
+ if (partitionResults.size() == 1) {
+ return partitionResults.get(0);
+ }
+
if (!needSorted) {
logger.debug("Using Iterators.concat to merge partition results");
return Iterators.concat(partitionResults.iterator());
}
- logger.debug("Using PartitionResultMerger to merge partition results");
- PartitionResultMerger merger = new PartitionResultMerger(partitionResults, info, context.getGroupKeyComparator());
- return merger.iterator();
+ logger.debug("Using SortMergedPartitionResultIterator to merge partition results");
+ return new SortMergedPartitionResultIterator(partitionResults, info, GTRecord.getComparator(groupByDims));
}
}
[09/35] kylin git commit: minor, fix whitespace in logging
Posted by li...@apache.org.
minor, fix whitespace in logging
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9147d3ed
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9147d3ed
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9147d3ed
Branch: refs/heads/master-hbase0.98
Commit: 9147d3edef5f650a3fce479da7968f39f9c63e84
Parents: 598b32e
Author: lidongsjtu <li...@apache.org>
Authored: Wed Mar 29 19:40:10 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Mar 29 19:40:10 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/CacheService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9147d3ed/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index af680a5..d3d6237 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -114,7 +114,7 @@ public class CacheService extends BasicService {
protected void cleanDataCache(String project) {
if (cacheManager != null) {
- logger.info("cleaning cache for project" + project + " (currently remove all entries)");
+ logger.info("cleaning cache for project " + project + " (currently remove all entries)");
cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll();
cacheManager.getCache(QueryService.EXCEPTION_QUERY_CACHE).removeAll();
} else {
[35/35] kylin git commit: KYLIN-2307 Create a branch for master with
HBase 0.98 API
Posted by li...@apache.org.
KYLIN-2307 Create a branch for master with HBase 0.98 API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/de4ddd14
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/de4ddd14
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/de4ddd14
Branch: refs/heads/master-hbase0.98
Commit: de4ddd14343c3edda324391430112f7f4188e727
Parents: 2e87fb4
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jan 23 13:17:37 2017 +0800
Committer: root <ro...@sandbox.hortonworks.com>
Committed: Sat Apr 8 11:54:26 2017 +0000
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 25 ++++
dev-support/test_all_against_hdp_2_4_0_0_169.sh | 25 ----
.../sandbox/capacity-scheduler.xml | 17 ++-
examples/test_case_data/sandbox/core-site.xml | 28 +---
examples/test_case_data/sandbox/hbase-site.xml | 119 +++++------------
examples/test_case_data/sandbox/hdfs-site.xml | 84 +++++-------
examples/test_case_data/sandbox/hive-site.xml | 89 +++++--------
examples/test_case_data/sandbox/mapred-site.xml | 57 +++------
examples/test_case_data/sandbox/yarn-site.xml | 127 +++----------------
.../kylin/provision/BuildCubeWithEngine.java | 17 +--
pom.xml | 117 +----------------
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 +++++++++++---
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 ++--
.../apache/kylin/rest/service/CubeService.java | 35 +++--
.../apache/kylin/rest/service/QueryService.java | 24 ++--
.../apache/kylin/rest/service/UserService.java | 17 +--
.../kylin/storage/hbase/HBaseConnection.java | 44 +++----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++--
.../storage/hbase/cube/SimpleHBaseStore.java | 20 +--
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 +--
.../storage/hbase/steps/DeprecatedGCStep.java | 24 ++--
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++--
.../storage/hbase/util/CleanHtableCLI.java | 12 +-
.../storage/hbase/util/CubeMigrationCLI.java | 37 +++---
.../hbase/util/CubeMigrationCheckCLI.java | 17 +--
.../hbase/util/DeployCoprocessorCLI.java | 27 ++--
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 ++---
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++-
.../hbase/util/HBaseRegionSizeCalculator.java | 35 +++--
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 ++---
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 +--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 +--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +-
.../storage/hbase/util/StorageCleanupJob.java | 20 ++-
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 +--
tool/pom.xml | 10 --
.../org/apache/kylin/tool/CubeMigrationCLI.java | 16 ++-
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
48 files changed, 595 insertions(+), 870 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
new file mode 100755
index 0000000..f7780dd
--- /dev/null
+++ b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+cd ${dir}
+cd ..
+
+mvn clean install -DskipTests 2>&1 | tee mci.log
+mvn verify -Dhdp.version=${HDP_VERSION:-"2.2.4.2-2"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/dev-support/test_all_against_hdp_2_4_0_0_169.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_4_0_0_169.sh b/dev-support/test_all_against_hdp_2_4_0_0_169.sh
deleted file mode 100755
index 2a3d24b..0000000
--- a/dev-support/test_all_against_hdp_2_4_0_0_169.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}
-cd ..
-
-mvn clean install -DskipTests 2>&1 | tee mci.log
-mvn verify -Dhdp.version=${HDP_VERSION:-"2.4.0.0-169"} -fae 2>&1 | tee mvnverify.log
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index e042aa5..7cb985c 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -47,6 +47,16 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.accessible-node-labels.default.maximum-capacity</name>
+ <value>-1</value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.acl_administer_queue</name>
<value>*</value>
</property>
@@ -57,6 +67,11 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.default-node-label-expression</name>
+ <value></value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
<value>*</value>
</property>
@@ -96,4 +111,4 @@
<value>default</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index a4ad5c6..0c5f62b 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -19,6 +19,7 @@
<property>
<name>fs.defaultFS</name>
<value>hdfs://sandbox.hortonworks.com:8020</value>
+ <final>true</final>
</property>
<property>
@@ -38,7 +39,7 @@
<property>
<name>hadoop.proxyuser.falcon.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -48,7 +49,7 @@
<property>
<name>hadoop.proxyuser.hbase.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
@@ -67,23 +68,13 @@
</property>
<property>
- <name>hadoop.proxyuser.hdfs.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.proxyuser.hdfs.hosts</name>
- <value>*</value>
- </property>
-
- <property>
<name>hadoop.proxyuser.hive.groups</name>
- <value>*</value>
+ <value>users</value>
</property>
<property>
<name>hadoop.proxyuser.hive.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <value>*</value>
</property>
<property>
@@ -132,15 +123,8 @@
</property>
<property>
- <name>hadoop.security.key.provider.path</name>
- <value></value>
- </property>
-
- <property>
<name>io.compression.codecs</name>
- <value>
- org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec
- </value>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 568de2e..46d5345 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -22,33 +22,8 @@
</property>
<property>
- <name>hbase.bucketcache.ioengine</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.percentage.in.combinedcache</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bucketcache.size</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.bulkload.staging.dir</name>
- <value>/apps/hbase/staging</value>
- </property>
-
- <property>
<name>hbase.client.keyvalue.maxsize</name>
- <value>1048576</value>
- </property>
-
- <property>
- <name>hbase.client.retries.number</name>
- <value>35</value>
+ <value>10485760</value>
</property>
<property>
@@ -63,19 +38,12 @@
<property>
<name>hbase.coprocessor.master.classes</name>
- <value>org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor</value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
- <value>
- org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint,org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor
- </value>
- </property>
-
- <property>
- <name>hbase.coprocessor.regionserver.classes</name>
- <value></value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
@@ -119,11 +87,6 @@
</property>
<property>
- <name>hbase.hstore.compaction.max</name>
- <value>10</value>
- </property>
-
- <property>
<name>hbase.hstore.compactionThreshold</name>
<value>3</value>
</property>
@@ -140,42 +103,32 @@
<property>
<name>hbase.master.info.port</name>
- <value>16010</value>
+ <value>60010</value>
</property>
<property>
<name>hbase.master.port</name>
- <value>16000</value>
+ <value>60000</value>
</property>
<property>
- <name>hbase.region.server.rpc.scheduler.factory.class</name>
- <value></value>
+ <name>hbase.regionserver.global.memstore.lowerLimit</name>
+ <value>0.38</value>
</property>
<property>
- <name>hbase.regionserver.global.memstore.size</name>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
<value>0.4</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
- <value>30</value>
+ <value>60</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
- <value>16030</value>
- </property>
-
- <property>
- <name>hbase.regionserver.port</name>
- <value>16020</value>
- </property>
-
- <property>
- <name>hbase.regionserver.wal.codec</name>
- <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value>
+ <value>60030</value>
</property>
<property>
@@ -184,26 +137,11 @@
</property>
<property>
- <name>hbase.rpc.controllerfactory.class</name>
- <value></value>
- </property>
-
- <property>
- <name>hbase.rpc.engine</name>
- <value>org.apache.hadoop.hbase.ipc.SecureRpcEngine</value>
- </property>
-
- <property>
<name>hbase.rpc.protection</name>
<value>PRIVACY</value>
</property>
<property>
- <name>hbase.rpc.timeout</name>
- <value>90000</value>
- </property>
-
- <property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
@@ -220,7 +158,7 @@
<property>
<name>hbase.tmp.dir</name>
- <value>/tmp/hbase-${user.name}</value>
+ <value>/hadoop/hbase</value>
</property>
<property>
@@ -240,27 +178,34 @@
<property>
<name>hfile.block.cache.size</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>phoenix.functions.allowUserDefinedFunctions</name>
- <value></value>
- </property>
-
- <property>
- <name>phoenix.query.timeoutMs</name>
- <value>60000</value>
+ <value>0.40</value>
</property>
<property>
<name>zookeeper.session.timeout</name>
- <value>60000</value>
+ <value>30000</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
-
-</configuration>
\ No newline at end of file
+ <property>
+ <name>hbase.client.pause</name>
+ <value>100</value>
+ <description>General client pause value. Used mostly as value to wait
+ before running a retry of a failed get, region lookup, etc.
+ See hbase.client.retries.number for description of how we backoff from
+ this initial pause amount and how this pause works w/ retries.</description>
+ </property>
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>5</value>
+ <description>Maximum retries. Used as maximum for all retryable
+ operations such as the getting of a cell's value, starting a row update,
+ etc. Retry interval is a rough function based on hbase.client.pause. At
+ first we retry at this interval but then with backoff, we pretty quickly reach
+ retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
+ ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index c06222e..d58f80d 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -22,7 +22,12 @@
<property>
<name>dfs.block.access.token.enable</name>
- <value>true</value>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.block.size</name>
+ <value>34217472</value>
</property>
<property>
@@ -46,21 +51,11 @@
</property>
<property>
- <name>dfs.client.retry.policy.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.cluster.administrators</name>
<value>hdfs</value>
</property>
<property>
- <name>dfs.content-summary.limit</name>
- <value>5000</value>
- </property>
-
- <property>
<name>dfs.datanode.address</name>
<value>0.0.0.0:50010</value>
</property>
@@ -73,6 +68,7 @@
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/hdfs/data</value>
+ <final>true</final>
</property>
<property>
@@ -88,6 +84,7 @@
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
+ <final>true</final>
</property>
<property>
@@ -111,18 +108,13 @@
</property>
<property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.encrypt.data.transfer.cipher.suites</name>
- <value>AES/CTR/NoPadding</value>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>1024</value>
</property>
<property>
- <name>dfs.encryption.key.provider.uri</name>
- <value></value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>
<property>
@@ -162,12 +154,7 @@
<property>
<name>dfs.namenode.accesstime.precision</name>
- <value>0</value>
- </property>
-
- <property>
- <name>dfs.namenode.audit.log.async</name>
- <value>true</value>
+ <value>3600000</value>
</property>
<property>
@@ -201,11 +188,6 @@
</property>
<property>
- <name>dfs.namenode.fslock.fair</name>
- <value>false</value>
- </property>
-
- <property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>
@@ -213,6 +195,7 @@
<property>
<name>dfs.namenode.http-address</name>
<value>sandbox.hortonworks.com:50070</value>
+ <final>true</final>
</property>
<property>
@@ -221,13 +204,9 @@
</property>
<property>
- <name>dfs.namenode.inode.attributes.provider.class</name>
- <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
- </property>
-
- <property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/hdfs/namenode</value>
+ <final>true</final>
</property>
<property>
@@ -236,13 +215,8 @@
</property>
<property>
- <name>dfs.namenode.rpc-address</name>
- <value>sandbox.hortonworks.com:8020</value>
- </property>
-
- <property>
<name>dfs.namenode.safemode.threshold-pct</name>
- <value>0.999</value>
+ <value>1.0f</value>
</property>
<property>
@@ -266,6 +240,16 @@
</property>
<property>
+ <name>dfs.nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs3.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+ <property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
@@ -277,7 +261,7 @@
<property>
<name>dfs.replication</name>
- <value>3</value>
+ <value>1</value>
</property>
<property>
@@ -288,11 +272,13 @@
<property>
<name>dfs.support.append</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
@@ -300,14 +286,4 @@
<value>022</value>
</property>
- <property>
- <name>nfs.exports.allowed.hosts</name>
- <value>* rw</value>
- </property>
-
- <property>
- <name>nfs.file.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
- </property>
-
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index a8c210e..1e78107 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -22,46 +22,11 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>Sandbox</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.maxThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.minThreads</name>
- <value>1</value>
- </property>
-
- <property>
- <name>atlas.hook.hive.synchronous</name>
- <value>true</value>
- </property>
-
- <property>
- <name>atlas.rest.address</name>
- <value>http://sandbox.hortonworks.com:21000</value>
- </property>
-
- <property>
- <name>datanucleus.autoCreateSchema</name>
- <value>false</value>
- </property>
-
- <property>
<name>datanucleus.cache.level2.type</name>
<value>none</value>
</property>
<property>
- <name>datanucleus.fixedDatastore</name>
- <value>true</value>
- </property>
-
- <property>
<name>hive.auto.convert.join</name>
<value>true</value>
</property>
@@ -73,7 +38,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>357913941</value>
+ <value>1000000000</value>
</property>
<property>
@@ -162,16 +127,6 @@
</property>
<property>
- <name>hive.default.fileformat</name>
- <value>TextFile</value>
- </property>
-
- <property>
- <name>hive.default.fileformat.managed</name>
- <value>TextFile</value>
- </property>
-
- <property>
<name>hive.enforce.bucketing</name>
<value>true</value>
</property>
@@ -207,6 +162,11 @@
</property>
<property>
+ <name>hive.exec.failure.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.max.created.files</name>
<value>100000</value>
</property>
@@ -237,11 +197,6 @@
</property>
<property>
- <name>hive.exec.orc.encoding.strategy</name>
- <value>SPEED</value>
- </property>
-
- <property>
<name>hive.exec.parallel</name>
<value>false</value>
</property>
@@ -252,6 +207,16 @@
</property>
<property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ </property>
+
+ <property>
<name>hive.exec.reducers.bytes.per.reducer</name>
<value>67108864</value>
</property>
@@ -297,6 +262,11 @@
</property>
<property>
+ <name>hive.heapsize</name>
+ <value>250</value>
+ </property>
+
+ <property>
<name>hive.limit.optimize.enable</name>
<value>true</value>
</property>
@@ -508,7 +478,7 @@
<property>
<name>hive.prewarm.numcontainers</name>
- <value>3</value>
+ <value>10</value>
</property>
<property>
@@ -518,7 +488,7 @@
<property>
<name>hive.security.authorization.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -538,7 +508,7 @@
<property>
<name>hive.security.metastore.authorization.manager</name>
- <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value>
+ <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly</value>
</property>
<property>
@@ -563,7 +533,12 @@
<property>
<name>hive.server2.enable.doAs</name>
- <value>false</value>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.impersonation</name>
+ <value>true</value>
</property>
<property>
@@ -573,7 +548,7 @@
<property>
<name>hive.server2.logging.operation.log.location</name>
- <value>/tmp/hive/operation_logs</value>
+ <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
</property>
<property>
@@ -678,7 +653,7 @@
<property>
<name>hive.tez.container.size</name>
- <value>1024</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index be470f9..e90f594 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>64</value>
+ <value>128</value>
</property>
<property>
@@ -27,13 +27,13 @@
</property>
<property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.map.memory.mb</name>
+ <value>512</value>
</property>
<property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>512</value>
</property>
<property>
@@ -48,9 +48,7 @@
<property>
<name>mapreduce.admin.user.env</name>
- <value>
- LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64
- </value>
+ <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
</property>
<property>
@@ -60,9 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>
- $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure
- </value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,18 +77,14 @@
</property>
<property>
- <name>mapreduce.job.counters.max</name>
- <value>130</value>
- </property>
-
- <property>
<name>mapreduce.job.emit-timeline-data</name>
<value>false</value>
</property>
+ <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
+ <value>1</value>
</property>
<property>
@@ -116,28 +108,13 @@
</property>
<property>
- <name>mapreduce.jobhistory.recovery.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.class</name>
- <value>org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.recovery.store.leveldb.path</name>
- <value>/hadoop/mapreduce/jhs</value>
- </property>
-
- <property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>sandbox.hortonworks.com:19888</value>
</property>
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx1228m</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -147,7 +124,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>1536</value>
+ <value>512</value>
</property>
<property>
@@ -182,7 +159,7 @@
<property>
<name>mapreduce.reduce.java.opts</name>
- <value>-Xmx1638m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -192,7 +169,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>2048</value>
+ <value>512</value>
</property>
<property>
@@ -242,7 +219,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>859</value>
+ <value>128</value>
</property>
<property>
@@ -257,7 +234,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx819m -Dhdp.version=${hdp.version}</value>
+ <value>-Xmx512m</value>
</property>
<property>
@@ -267,7 +244,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>1024</value>
+ <value>512</value>
</property>
<property>
@@ -275,4 +252,4 @@
<value>/user</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index ebdf44a..8256158 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -18,7 +18,7 @@
<property>
<name>hadoop.registry.rm.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -28,29 +28,22 @@
<property>
<name>yarn.acl.enable</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
<name>yarn.admin.acl</name>
- <value>*</value>
+ <value></value>
</property>
<property>
<name>yarn.application.classpath</name>
- <value>
- $HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*
- </value>
- </property>
-
- <property>
- <name>yarn.authorization-provider</name>
- <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
+ <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
</property>
<property>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -79,11 +72,6 @@
</property>
<property>
- <name>yarn.node-labels.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.node-labels.fs-store.retry-policy-spec</name>
<value>2000, 500</value>
</property>
@@ -94,6 +82,11 @@
</property>
<property>
+ <name>yarn.node-labels.manager-class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.address</name>
<value>0.0.0.0:45454</value>
</property>
@@ -105,7 +98,7 @@
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle,spark_shuffle</value>
+ <value>mapreduce_shuffle</value>
</property>
<property>
@@ -114,11 +107,6 @@
</property>
<property>
- <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
- <value>org.apache.spark.network.yarn.YarnShuffleService</value>
- </property>
-
- <property>
<name>yarn.nodemanager.bind-host</name>
<value>0.0.0.0</value>
</property>
@@ -160,7 +148,7 @@
<property>
<name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>120000</value>
+ <value>60000</value>
</property>
<property>
@@ -255,12 +243,12 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
- <value>7168</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>80</value>
+ <value>100</value>
</property>
<property>
@@ -349,11 +337,6 @@
</property>
<property>
- <name>yarn.resourcemanager.scheduler.monitor.enable</name>
- <value>false</value>
- </property>
-
- <property>
<name>yarn.resourcemanager.state-store.max-completed-applications</name>
<value>${yarn.resourcemanager.max-completed-applications}</value>
</property>
@@ -385,7 +368,7 @@
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8090</value>
+ <value>localhost:8090</value>
</property>
<property>
@@ -425,7 +408,7 @@
<property>
<name>yarn.resourcemanager.zk-address</name>
- <value>sandbox.hortonworks.com:2181</value>
+ <value>localhost:2181</value>
</property>
<property>
@@ -450,22 +433,12 @@
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
- <value>7168</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-vcores</name>
- <value>3</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>yarn.scheduler.minimum-allocation-vcores</name>
- <value>1</value>
+ <value>1536</value>
</property>
<property>
@@ -494,41 +467,6 @@
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
- <value>/ats/active/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
- <value>3600</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
- <value>/ats/done/</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
- <value>org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
- <value>60</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
- <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.generic-application-history.store-class</name>
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
@@ -544,11 +482,6 @@
</property>
<property>
- <name>yarn.timeline-service.leveldb-state-store.path</name>
- <value>/hadoop/yarn/timeline</value>
- </property>
-
- <property>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
<value>/hadoop/yarn/timeline</value>
</property>
@@ -574,23 +507,8 @@
</property>
<property>
- <name>yarn.timeline-service.plugin.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.recovery.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.state-store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
- </property>
-
- <property>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
@@ -604,11 +522,6 @@
</property>
<property>
- <name>yarn.timeline-service.version</name>
- <value>1.5</value>
- </property>
-
- <property>
<name>yarn.timeline-service.webapp.address</name>
<value>sandbox.hortonworks.com:8188</value>
</property>
@@ -618,4 +531,4 @@
<value>sandbox.hortonworks.com:8190</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index df6c45e..d819098 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -32,9 +32,11 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,7 +60,6 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -95,10 +96,10 @@ public class BuildCubeWithEngine {
logger.error("error", e);
exitCode = 1;
}
-
+
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithEngine.class.getName());
-
+
System.exit(exitCode);
}
@@ -359,10 +360,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
- String tableName = segment.getStorageLocationIdentifier();
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
+ String tableName = segment.getStorageLocationIdentifier();
+ try (HTable table = new HTable(conf, tableName)) {
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 40ccd0c..4d6c436 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,15 +46,15 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0</hadoop2.version>
+ <yarn.version>2.6.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>0.14.0</hive.version>
+ <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
<!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
@@ -71,7 +71,7 @@
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.8</zookeeper.version>
- <curator.version>2.7.1</curator.version>
+ <curator.version>2.6.0</curator.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
<jsch.version>0.1.53</jsch.version>
@@ -842,11 +842,6 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
-
- <repository>
- <id>cloudera</id>
- <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
- </repository>
</repositories>
<build>
@@ -1200,106 +1195,6 @@
</build>
</profile>
<profile>
- <id>cdh5.7</id>
- <properties>
- <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
- <yarn.version>2.6.0-cdh5.7.0</yarn.version>
- <hive.version>1.1.0-cdh5.7.0</hive.version>
- <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
- <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
- <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <fork>true</fork>
- <meminitial>1024m</meminitial>
- <maxmem>2048m</maxmem>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-jamm</id>
- <goals>
- <goal>copy</goal>
- </goals>
- <phase>generate-test-resources</phase>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>com.github.jbellis</groupId>
- <artifactId>jamm</artifactId>
- <outputDirectory>${project.build.testOutputDirectory}</outputDirectory>
- <destFileName>jamm.jar</destFileName>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.jacoco</groupId>
- <artifactId>jacoco-maven-plugin</artifactId>
- <configuration>
- <append>true</append>
- <destFile>
- ${sonar.jacoco.reportPath}
- </destFile>
- </configuration>
- <executions>
- <execution>
- <id>pre-test</id>
- <goals>
- <goal>prepare-agent</goal>
- </goals>
- <configuration>
- <propertyName>surefireArgLine</propertyName>
- </configuration>
- </execution>
- <execution>
- <id>post-test</id>
- <phase>test</phase>
- <goals>
- <goal>report</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.19.1</version>
- <configuration>
- <reportsDirectory>${project.basedir}/../target/surefire-reports</reportsDirectory>
- <excludes>
- <exclude>**/IT*.java</exclude>
- </excludes>
- <systemProperties>
- <property>
- <name>buildCubeUsingProvidedData</name>
- <value>false</value>
- </property>
- <property>
- <name>log4j.configuration</name>
- <value>file:${project.basedir}/../build/conf/kylin-tools-log4j.properties</value>
- </property>
- </systemProperties>
- <argLine>-javaagent:${project.build.testOutputDirectory}/jamm.jar ${argLine} ${surefireArgLine}</argLine>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<!-- This profile adds/overrides few features of the 'apache-release'
profile in the parent pom. -->
<id>apache-release</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 8095bf8..ea68855 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- Table getTable(String tableName) throws IOException;
+ HTableInterface getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index cc76b87..d9326f5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private Table mockedAclTable;
- private Table mockedUserTable;
+ private HTableInterface mockedAclTable;
+ private HTableInterface mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index 972eea9..d0aa0ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements Table {
+public class MockHTable implements HTableInterface {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,6 +114,14 @@ public class MockHTable implements Table {
this.columnFamilies.add(columnFamily);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public byte[] getTableName() {
+ return tableName.getBytes();
+ }
+
@Override
public TableName getName() {
return null;
@@ -192,8 +200,8 @@ public class MockHTable implements Table {
}
@Override
- public boolean[] existsAll(List<Get> list) throws IOException {
- return new boolean[0];
+ public Boolean[] exists(List<Get> gets) throws IOException {
+ return new Boolean[0];
}
/**
@@ -298,6 +306,15 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
+ // FIXME: implement
+ return null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -429,7 +446,7 @@ public class MockHTable implements Table {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow();
+ filter.filterRow(nkvs);
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -518,11 +535,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -543,7 +555,7 @@ public class MockHTable implements Table {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDelete()) {
+ if (kv.isDeleteFamily()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -580,11 +592,6 @@ public class MockHTable implements Table {
return false;
}
- @Override
- public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
- return false;
- }
-
/**
* {@inheritDoc}
*/
@@ -598,7 +605,7 @@ public class MockHTable implements Table {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, null);
+ return incrementColumnValue(row, family, qualifier, amount, true);
}
@Override
@@ -610,6 +617,37 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
+ if (check(row, family, qualifier, null)) {
+ Put put = new Put(row);
+ put.add(family, qualifier, Bytes.toBytes(amount));
+ put(put);
+ return amount;
+ }
+ long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
+ data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
+ return newValue;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isAutoFlush() {
+ return true;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void flushCommits() throws IOException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public void close() throws IOException {
}
@@ -635,6 +673,29 @@ public class MockHTable implements Table {
* {@inheritDoc}
*/
@Override
+ public void setAutoFlush(boolean autoFlush) {
+ throw new NotImplementedException();
+
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
+ throw new NotImplementedException();
+
+ }
+
+ @Override
+ public void setAutoFlushTo(boolean autoFlush) {
+ throw new NotImplementedException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index d1a1384..1d520c4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,8 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -59,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public Table getTable(String tableName) throws IOException {
+ public HTableInterface getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ return HBaseConnection.get(hbaseUrl).getTable(userTableName);
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 3e3efec..d693a67 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- Table htable = null;
+ HTableInterface htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,16 +226,17 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -249,7 +250,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -265,6 +266,7 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
+ htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -282,7 +284,7 @@ public class AclService implements MutableAclService {
throw e;
}
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -293,16 +295,17 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
+ htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a6310d1..cb554c0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -27,7 +27,9 @@ import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -393,24 +395,33 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
- Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
+
+ Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
+ HTable table = null;
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ try {
+ table = new HTable(hconf, tableName);
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- regionCount = sizeMap.size();
+ for (long s : sizeMap.values()) {
+ tableSize += s;
+ }
+
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 6a38638..8972456 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -48,11 +48,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -163,13 +163,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -195,13 +196,14 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- Table htable = null;
+ HTableInterface htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
Put put = new Put(Bytes.toBytes(creator));
- put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
+ htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +215,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- Table htable = null;
+ HTableInterface htable = null;
try {
- org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
+ HConnection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
+ htable = conn.getTable(userTableName);
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index ab54882..07c7c6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
-
- put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+ put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,13 +163,14 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
+ htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -184,7 +185,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- Table htable = null;
+ HTableInterface htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -215,7 +216,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- Table htable = null;
+ HTableInterface htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 73f31c5..2351412 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -39,9 +39,9 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -63,7 +63,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
+ private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -74,7 +74,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (Connection conn : connPool.values()) {
+ for (HConnection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -143,7 +143,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -207,9 +207,9 @@ public class HBaseConnection {
// ============================================================================
- // returned Connection can be shared by multiple threads and does not require close()
+ // returned HConnection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static Connection get(String url) {
+ public static HConnection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -217,13 +217,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- Connection connection = connPool.get(url);
+ HConnection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = ConnectionFactory.createConnection(conf);
+ connection = HConnectionManager.createConnection(conf);
connPool.put(url, connection);
}
@@ -242,8 +242,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static boolean tableExists(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -263,18 +263,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
- Admin hbase = conn.getAdmin();
- TableName tableName = TableName.valueOf(table);
+ public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
+
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(tableName, newFamilyDescriptor(family));
+ hbase.addColumn(table, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -327,8 +327,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(Connection conn, String tableName) throws IOException {
- Admin hbase = conn.getAdmin();
+ public static void deleteTable(HConnection conn, String tableName) throws IOException {
+ HBaseAdmin hbase = new HBaseAdmin(conn);
try {
if (!tableExists(conn, tableName)) {
@@ -338,10 +338,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
- hbase.disableTable(TableName.valueOf(tableName));
+ if (hbase.isTableEnabled(tableName)) {
+ hbase.disableTable(tableName);
}
- hbase.deleteTable(TableName.valueOf(tableName));
+ hbase.deleteTable(tableName);
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/de4ddd14/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 74ab017..a44de4f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -30,15 +30,14 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +68,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- Connection getConnection() throws IOException {
+ HConnection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -121,7 +120,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,12 +237,13 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
+ table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -251,7 +251,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -264,6 +264,8 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
+ table.flushCommits();
+
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -272,7 +274,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -285,6 +287,7 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
+ table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -305,7 +308,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
+ HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -314,7 +317,7 @@ public class HBaseResourceStore extends ResourceStore {
}
- private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -333,7 +336,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
@@ -358,7 +361,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -366,8 +369,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.addColumn(B_FAMILY, B_COLUMN, content);
- put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.add(B_FAMILY, B_COLUMN, content);
+ put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
[02/35] kylin git commit: #KYLIN-490 support multiple column distinct
count
Posted by li...@apache.org.
#KYLIN-490 support multiple column distinct count
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/636282db
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/636282db
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/636282db
Branch: refs/heads/master-hbase0.98
Commit: 636282db889973fe29269b43e417414effb68b76
Parents: f72a3f6
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Mar 22 19:22:22 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Mar 27 15:36:07 2017 +0800
----------------------------------------------------------------------
.../BitmapIntersectDistinctCountAggFunc.java | 9 +-
.../measure/percentile/PercentileAggFunc.java | 9 +-
.../kylin/metadata/model/FunctionDesc.java | 62 ++++++---
.../kylin/metadata/model/ParameterDesc.java | 135 +++++++++++++++++--
.../kylin/query/relnode/OLAPAggregateRel.java | 86 +++++++++---
5 files changed, 250 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/636282db/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
index cd4d306..a1e2665 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapIntersectDistinctCountAggFunc.java
@@ -17,6 +17,8 @@
*/
package org.apache.kylin.measure.bitmap;
+import org.apache.kylin.measure.ParamAsMeasureCount;
+
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -27,9 +29,14 @@ import java.util.Map;
* Example: intersect_count(uuid, event, array['A', 'B', 'C']), meaning find the count of uuid in all A/B/C 3 bitmaps
* requires an bitmap count distinct measure of uuid, and an dimension of event
*/
-public class BitmapIntersectDistinctCountAggFunc {
+public class BitmapIntersectDistinctCountAggFunc implements ParamAsMeasureCount {
private static final BitmapCounterFactory factory = RoaringBitmapCounterFactory.INSTANCE;
+ @Override
+ public int getParamAsMeasureCount() {
+ return -2;
+ }
+
public static class RetentionPartialResult {
Map<Object, BitmapCounter> map;
List keyList;
http://git-wip-us.apache.org/repos/asf/kylin/blob/636282db/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
index ad02019..d3cec8f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileAggFunc.java
@@ -18,7 +18,9 @@
package org.apache.kylin.measure.percentile;
-public class PercentileAggFunc {
+import org.apache.kylin.measure.ParamAsMeasureCount;
+
+public class PercentileAggFunc implements ParamAsMeasureCount{
public static PercentileCounter init() {
return null;
}
@@ -41,4 +43,9 @@ public class PercentileAggFunc {
public static double result(PercentileCounter counter) {
return counter == null ? 0L : counter.getResultEstimate();
}
+
+ @Override
+ public int getParamAsMeasureCount() {
+ return 1;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636282db/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index cbd7574..61c5fac 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -18,22 +18,26 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.basic.BasicMeasureType;
+import org.apache.kylin.metadata.datatype.DataType;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.measure.basic.BasicMeasureType;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
/**
*/
@@ -48,7 +52,7 @@ public class FunctionDesc implements Serializable {
r.returnDataType = DataType.getType(returnType);
return r;
}
-
+
public static final String FUNC_SUM = "SUM";
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
@@ -95,7 +99,7 @@ public class FunctionDesc implements Serializable {
}
}
- if(parameter != null)
+ if (parameter != null)
parameter.setColRefs(colRefs);
}
@@ -140,6 +144,8 @@ public class FunctionDesc implements Serializable {
return getParameter().getValue();
} else if (isCount()) {
return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
+ } else if (isCountDistinct()) {
+ return "_KY_" + getFullExpressionInAlphabetOrder().replaceAll("[(),. ]", "_");
} else {
return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_");
}
@@ -197,6 +203,25 @@ public class FunctionDesc implements Serializable {
return sb.toString();
}
+ /**
+ * Parameters' name appears in alphabet order.
+ * This method is used for funcs whose parameters appear in arbitrary order
+ */
+ public String getFullExpressionInAlphabetOrder() {
+ StringBuilder sb = new StringBuilder(expression);
+ sb.append("(");
+ ParameterDesc localParam = parameter;
+ List<String> flatParams = Lists.newArrayList();
+ while (localParam != null) {
+ flatParams.add(localParam.getValue());
+ localParam = localParam.getNextParameter();
+ }
+ Collections.sort(flatParams);
+ sb.append(Joiner.on(",").join(flatParams));
+ sb.append(")");
+ return sb.toString();
+ }
+
public boolean isDimensionAsMetric() {
return isDimensionAsMetric;
}
@@ -264,13 +289,20 @@ public class FunctionDesc implements Serializable {
return false;
} else if (!expression.equals(other.expression))
return false;
- // NOTE: don't check the parameter of count()
- if (isCount() == false) {
+ if (isCountDistinct()) {
+ // for count distinct func, param's order doesn't matter
+ if (parameter == null) {
+ if (other.parameter != null)
+ return false;
+ } else {
+ return parameter.equalInArbitraryOrder(other.parameter);
+ }
+ } else if (!isCount()) { // NOTE: don't check the parameter of count()
if (parameter == null) {
if (other.parameter != null)
return false;
} else {
- if (!parameter.equals(other.parameter))
+ if (!parameter.equals(other.parameter))
return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636282db/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 8ad20a8..5ba2f14 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -18,17 +18,19 @@
package org.apache.kylin.metadata.model;
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-
-import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.List;
+import com.google.common.collect.Sets;
/**
*/
@@ -38,9 +40,9 @@ public class ParameterDesc implements Serializable {
public static ParameterDesc newInstance(Object... objs) {
if (objs.length == 0)
throw new IllegalArgumentException();
-
+
ParameterDesc r = new ParameterDesc();
-
+
Object obj = objs[0];
if (obj instanceof TblColRef) {
TblColRef col = (TblColRef) obj;
@@ -51,7 +53,7 @@ public class ParameterDesc implements Serializable {
r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT;
r.value = (String) obj;
}
-
+
if (objs.length >= 2) {
r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1, objs.length));
if (r.nextParameter.colRefs.size() > 0) {
@@ -63,7 +65,7 @@ public class ParameterDesc implements Serializable {
}
return r;
}
-
+
@JsonProperty("type")
private String type;
@JsonProperty("value")
@@ -74,6 +76,15 @@ public class ParameterDesc implements Serializable {
private ParameterDesc nextParameter;
private List<TblColRef> colRefs = ImmutableList.of();
+ private Set<PlainParameter> plainParameters = null;
+
+ // Lazy evaluation
+ public Set<PlainParameter> getPlainParameters() {
+ if (plainParameters == null) {
+ plainParameters = PlainParameter.createFromParameterDesc(this);
+ }
+ return plainParameters;
+ }
public String getType() {
return type;
@@ -86,7 +97,7 @@ public class ParameterDesc implements Serializable {
public String getValue() {
return value;
}
-
+
void setValue(String value) {
this.value = value;
}
@@ -94,7 +105,7 @@ public class ParameterDesc implements Serializable {
public List<TblColRef> getColRefs() {
return colRefs;
}
-
+
void setColRefs(List<TblColRef> colRefs) {
this.colRefs = colRefs;
}
@@ -118,7 +129,7 @@ public class ParameterDesc implements Serializable {
if (type != null ? !type.equals(that.type) : that.type != null)
return false;
-
+
ParameterDesc p = this, q = that;
int refi = 0, refj = 0;
for (; p != null && q != null; p = p.nextParameter, q = q.nextParameter) {
@@ -138,10 +149,24 @@ public class ParameterDesc implements Serializable {
return false;
}
}
-
+
return p == null && q == null;
}
+ public boolean equalInArbitraryOrder(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ParameterDesc that = (ParameterDesc) o;
+
+ Set<PlainParameter> thisPlainParams = this.getPlainParameters();
+ Set<PlainParameter> thatPlainParams = that.getPlainParameters();
+
+ return thisPlainParams.containsAll(thatPlainParams) && thatPlainParams.containsAll(thisPlainParams);
+ }
+
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
@@ -154,4 +179,88 @@ public class ParameterDesc implements Serializable {
return "ParameterDesc [type=" + type + ", value=" + value + ", nextParam=" + nextParameter + "]";
}
+ /**
+ * PlainParameter is created to present ParameterDesc in List style.
+ * Compared to ParameterDesc its advantage is:
+ * 1. easy to compare without considering order
+ * 2. easy to compare one by one
+ */
+ private static class PlainParameter {
+ private String type;
+ private String value;
+ private TblColRef colRef = null;
+
+ private PlainParameter() {
+ }
+
+ public boolean isColumnType() {
+ return FunctionDesc.PARAMETER_TYPE_COLUMN.equals(type);
+ }
+
+ static Set<PlainParameter> createFromParameterDesc(ParameterDesc parameterDesc) {
+ Set<PlainParameter> result = Sets.newHashSet();
+ ParameterDesc local = parameterDesc;
+ List<TblColRef> totalColRef = parameterDesc.colRefs;
+ Integer colIndex = 0;
+ while (local != null) {
+ if (local.isColumnType()) {
+ result.add(createSingleColumnParameter(local, totalColRef.get(colIndex++)));
+ } else {
+ result.add(createSingleValueParameter(local));
+ }
+ local = local.nextParameter;
+ }
+ return result;
+ }
+
+ static PlainParameter createSingleValueParameter(ParameterDesc parameterDesc) {
+ PlainParameter single = new PlainParameter();
+ single.type = parameterDesc.type;
+ single.value = parameterDesc.value;
+ return single;
+ }
+
+ static PlainParameter createSingleColumnParameter(ParameterDesc parameterDesc, TblColRef colRef) {
+ PlainParameter single = new PlainParameter();
+ single.type = parameterDesc.type;
+ single.value = parameterDesc.value;
+ single.colRef = colRef;
+ return single;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = type != null ? type.hashCode() : 0;
+ result = 31 * result + (colRef != null ? colRef.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ PlainParameter that = (PlainParameter) o;
+
+ if (type != null ? !type.equals(that.type) : that.type != null)
+ return false;
+
+ if (this.isColumnType()) {
+ if (!that.isColumnType())
+ return false;
+ if (!this.colRef.equals(that.colRef)) {
+ return false;
+ }
+ } else {
+ if (that.isColumnType())
+ return false;
+ if (!this.value.equals(that.value))
+ return false;
+ }
+
+ return true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/636282db/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 8d7c597..2c75a14 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -18,6 +18,7 @@
package org.apache.kylin.query.relnode;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -55,6 +56,7 @@ import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.ParamAsMeasureCount;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
@@ -71,6 +73,7 @@ import com.google.common.collect.Sets;
public class OLAPAggregateRel extends Aggregate implements OLAPRel {
private final static Map<String, String> AGGR_FUNC_MAP = new HashMap<String, String>();
+ private final static Map<String, Integer> AGGR_FUNC_PARAM_AS_MEASTURE_MAP = new HashMap<String, Integer>();
static {
AGGR_FUNC_MAP.put("SUM", "SUM");
@@ -84,6 +87,15 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
for (String udaf : udafFactories.keySet()) {
AGGR_FUNC_MAP.put(udaf, udafFactories.get(udaf).getAggrFunctionName());
}
+
+ Map<String, Class<?>> udafs = MeasureTypeFactory.getUDAFs();
+ for (String func : udafs.keySet()) {
+ try {
+ AGGR_FUNC_PARAM_AS_MEASTURE_MAP.put(func, ((ParamAsMeasureCount) (udafs.get(func).newInstance())).getParamAsMeasureCount());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
private static String getSqlFuncName(AggregateCall aggCall) {
@@ -235,12 +247,27 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.aggregations = new ArrayList<FunctionDesc>();
for (AggregateCall aggCall : this.rewriteAggCalls) {
ParameterDesc parameter = null;
+ // By default all args are included, UDFs can define their own in getParamAsMeasureCount method.
if (!aggCall.getArgList().isEmpty()) {
- // TODO: Currently only get the column of first param
- int index = aggCall.getArgList().get(0);
- TblColRef column = inputColumnRowType.getColumnByIndex(index);
- if (!column.isInnerColumn()) {
- parameter = ParameterDesc.newInstance(column);
+ List<TblColRef> columns = Lists.newArrayList();
+ String funcName = getSqlFuncName(aggCall);
+ int columnsCount = aggCall.getArgList().size();
+ if (AGGR_FUNC_PARAM_AS_MEASTURE_MAP.containsKey(funcName)) {
+ int asMeasureCnt = AGGR_FUNC_PARAM_AS_MEASTURE_MAP.get(funcName);
+ if (asMeasureCnt > 0) {
+ columnsCount = asMeasureCnt;
+ } else {
+ columnsCount += asMeasureCnt;
+ }
+ }
+ for (Integer index : aggCall.getArgList().subList(0, columnsCount)) {
+ TblColRef column = inputColumnRowType.getColumnByIndex(index);
+ if (!column.isInnerColumn()) {
+ columns.add(column);
+ }
+ }
+ if (!columns.isEmpty()) {
+ parameter = ParameterDesc.newInstance(columns.toArray(new TblColRef[columns.size()]));
}
}
String expression = getAggrFuncName(aggCall);
@@ -341,10 +368,11 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
AggregateCall aggCall = this.rewriteAggCalls.get(i);
if (!aggCall.getArgList().isEmpty()) {
- int index = aggCall.getArgList().get(0);
- TblColRef column = inputColumnRowType.getColumnByIndex(index);
- if (!column.isInnerColumn()) {
- this.context.metricsColumns.add(column);
+ for (Integer index : aggCall.getArgList()) {
+ TblColRef column = inputColumnRowType.getColumnByIndex(index);
+ if (!column.isInnerColumn()) {
+ this.context.metricsColumns.add(column);
+ }
}
}
}
@@ -385,18 +413,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
return aggCall;
}
- // rebuild parameters
- List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
- if (func.needRewriteField()) {
- RelDataTypeField field = getInput().getRowType().getField(func.getRewriteFieldName(), true, false);
- if (newArgList.isEmpty()) {
- newArgList.add(field.getIndex());
- } else {
- // only the first column got overwritten
- newArgList.set(0, field.getIndex());
- }
- }
-
// rebuild function
String callName = getSqlFuncName(aggCall);
RelDataType fieldType = aggCall.getType();
@@ -408,12 +424,40 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
newAgg = createCustomAggFunction(callName, fieldType, udafMap.get(callName));
}
+ // rebuild parameters
+ List<Integer> newArgList = Lists.newArrayList(aggCall.getArgList());
+ if (udafMap != null && udafMap.containsKey(callName)) {
+ newArgList = truncArgList(newArgList, udafMap.get(callName));
+ }
+ if (func.needRewriteField()) {
+ RelDataTypeField field = getInput().getRowType().getField(func.getRewriteFieldName(), true, false);
+ if (newArgList.isEmpty()) {
+ newArgList.add(field.getIndex());
+ } else {
+ // TODO: only the first column got overwritten
+ newArgList.set(0, field.getIndex());
+ }
+ }
+
// rebuild aggregate call
@SuppressWarnings("deprecation")
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, callName);
return newAggCall;
}
+ /**
+ * truncate Arg List according to UDAF's "add" method parameter count
+ */
+ private List<Integer> truncArgList(List<Integer> argList, Class<?> udafClazz) {
+ int argListLength = argList.size();
+ for (Method method : udafClazz.getMethods()) {
+ if (method.getName().equals("add")) {
+ argListLength = Math.min(method.getParameterTypes().length - 1, argListLength);
+ }
+ }
+ return argList.subList(0, argListLength);
+ }
+
private SqlAggFunction createCustomAggFunction(String funcName, RelDataType returnType, Class<?> customAggFuncClz) {
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
SqlIdentifier sqlIdentifier = new SqlIdentifier(funcName, new SqlParserPos(1, 1));
[07/35] kylin git commit: KYLIN-2518 Optimize put row key to hll
Posted by li...@apache.org.
KYLIN-2518 Optimize put row key to hll
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c218214
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c218214
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c218214
Branch: refs/heads/master-hbase0.98
Commit: 4c21821471cb261cfecdf8289c5f8284af817b3e
Parents: b1cc0dd
Author: xiefan46 <95...@qq.com>
Authored: Mon Mar 27 18:13:03 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 29 11:01:24 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/measure/hllc/HLLCounter.java | 54 ++--
.../mr/steps/FactDistinctColumnsMapper.java | 31 +-
.../mr/steps/NewCubeSamplingMethodTest.java | 299 +++++++++++++++++++
3 files changed, 341 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c218214/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
index 82c881b..b793465 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
@@ -60,7 +60,7 @@ public class HLLCounter implements Serializable, Comparable<HLLCounter> {
merge(another);
}
- HLLCounter(int p, RegisterType type) {
+ public HLLCounter(int p, RegisterType type) {
this(p, type, Hashing.murmur3_128());
}
@@ -99,6 +99,10 @@ public class HLLCounter implements Serializable, Comparable<HLLCounter> {
add(hashFunc.hashBytes(value, offset, length).asLong());
}
+ public void addHashDirectly(long hash){
+ add(hash);
+ }
+
protected void add(long hash) {
int bucketMask = m - 1;
int bucket = (int) (hash & bucketMask);
@@ -126,7 +130,7 @@ public class HLLCounter implements Serializable, Comparable<HLLCounter> {
}
private void toDenseIfNeeded() {
- if (register instanceof SparseRegister) {
+ if (register.getRegisterType() == RegisterType.SPARSE) {
if (isDense(register.getSize())) {
register = ((SparseRegister) register).toDense(p);
}
@@ -137,36 +141,36 @@ public class HLLCounter implements Serializable, Comparable<HLLCounter> {
assert this.p == another.p;
assert this.hashFunc == another.hashFunc;
switch (register.getRegisterType()) {
- case SINGLE_VALUE:
- switch (another.getRegisterType()) {
case SINGLE_VALUE:
- if (register.getSize() > 0 && another.register.getSize() > 0) {
- register = ((SingleValueRegister) register).toSparse();
- } else {
- SingleValueRegister sr = (SingleValueRegister) another.register;
- if (sr.getSize() > 0)
- register.set(sr.getSingleValuePos(), sr.getValue());
- return;
+ switch (another.getRegisterType()) {
+ case SINGLE_VALUE:
+ if (register.getSize() > 0 && another.register.getSize() > 0) {
+ register = ((SingleValueRegister) register).toSparse();
+ } else {
+ SingleValueRegister sr = (SingleValueRegister) another.register;
+ if (sr.getSize() > 0)
+ register.set(sr.getSingleValuePos(), sr.getValue());
+ return;
+ }
+ break;
+ case SPARSE:
+ register = ((SingleValueRegister) register).toSparse();
+ break;
+ case DENSE:
+ register = ((SingleValueRegister) register).toDense(this.p);
+ break;
+ default:
+ break;
}
+
break;
case SPARSE:
- register = ((SingleValueRegister) register).toSparse();
- break;
- case DENSE:
- register = ((SingleValueRegister) register).toDense(this.p);
+ if (another.getRegisterType() == RegisterType.DENSE) {
+ register = ((SparseRegister) register).toDense(p);
+ }
break;
default:
break;
- }
-
- break;
- case SPARSE:
- if (another.getRegisterType() == RegisterType.DENSE) {
- register = ((SparseRegister) register).toDense(p);
- }
- break;
- default:
- break;
}
register.merge(another.register);
toDenseIfNeeded();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c218214/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 9f65163..e6cea2b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -24,13 +24,13 @@ import java.util.Collection;
import java.util.List;
import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.hllc.RegisterType;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -62,7 +62,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private HashFunction hf = null;
private int rowCount = 0;
private int samplingPercentage;
- private ByteArray[] row_hashcodes = null;
+ //private ByteArray[] row_hashcodes = null;
+ private long[] rowHashCodesLong = null;
private ByteBuffer tmpbuf;
private static final Text EMPTY_TEXT = new Text();
public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE;
@@ -92,14 +93,11 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
allCuboidsHLL = new HLLCounter[cuboidIds.length];
for (int i = 0; i < cuboidIds.length; i++) {
- allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision());
+ allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision(), RegisterType.DENSE);
}
- hf = Hashing.murmur3_32();
- row_hashcodes = new ByteArray[nRowKey];
- for (int i = 0; i < nRowKey; i++) {
- row_hashcodes[i] = new ByteArray();
- }
+ hf = Hashing.murmur3_128();
+ rowHashCodesLong = new long[nRowKey];
TblColRef partitionColRef = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (partitionColRef != null) {
@@ -211,26 +209,23 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
private void putRowKeyToHLL(String[] row) {
-
//generate hash for each row key column
for (int i = 0; i < nRowKey; i++) {
Hasher hc = hf.newHasher();
String colValue = row[intermediateTableDesc.getRowKeyColumnIndexes()[i]];
- if (colValue != null) {
- row_hashcodes[i].set(hc.putString(colValue).hash().asBytes());
- } else {
- row_hashcodes[i].set(hc.putInt(0).hash().asBytes());
- }
+ if (colValue == null)
+ colValue = "0";
+ byte[] bytes = hc.putString(colValue).hash().asBytes();
+ rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash value to distinguish between (a,b) and (b,a)
}
// user the row key column hash to get a consolidated hash for each cuboid
for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
- Hasher hc = hf.newHasher();
+ long value = 0;
for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
- hc.putBytes(row_hashcodes[allCuboidsBitSet[i][position]].array());
+ value += rowHashCodesLong[allCuboidsBitSet[i][position]];
}
-
- allCuboidsHLL[i].add(hc.hash().asBytes());
+ allCuboidsHLL[i].addHashDirectly(value);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c218214/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
new file mode 100644
index 0000000..f018f28
--- /dev/null
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.engine.mr.steps;
+
+import com.google.common.collect.Lists;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+@Ignore
+public class NewCubeSamplingMethodTest {
+
+ private static final int ROW_LENGTH = 10;
+
+ private Integer[][] allCuboidsBitSet;
+
+ private long baseCuboidId;
+
+ private final int rowCount = 500000;
+
+ @Before
+ public void setup() {
+ baseCuboidId = (1L << ROW_LENGTH) - 1;
+ createAllCuboidBitSet();
+ System.out.println("Totally have " + allCuboidsBitSet.length + " cuboids.");
+ }
+
+ @Ignore
+ @Test
+ public void testRandomData() throws Exception {
+ List<List<String>> dataSet = getRandomDataset(rowCount);
+ comparePerformanceBasic(dataSet);
+ compareAccuracyBasic(dataSet);
+ }
+
+
+ @Ignore
+ @Test
+ public void testSmallCardData() throws Exception {
+ List<List<String>> dataSet = getSmallCardDataset(rowCount);
+ comparePerformanceBasic(dataSet);
+ compareAccuracyBasic(dataSet);
+ }
+
+
+ public void comparePerformanceBasic(final List<List<String>> rows) throws Exception {
+ //old hash method
+ ByteArray[] colHashValues = getNewColHashValues(ROW_LENGTH);
+ HLLCounter[] cuboidCounters = getNewCuboidCounters(allCuboidsBitSet.length);
+ long start = System.currentTimeMillis();
+ for (List<String> row : rows) {
+ putRowKeyToHLL(row, colHashValues, cuboidCounters, Hashing.murmur3_32());
+ }
+ long totalTime = System.currentTimeMillis() - start;
+ System.out.println("old method cost time : " + totalTime);
+ //new hash method
+ colHashValues = getNewColHashValues(ROW_LENGTH);
+ cuboidCounters = getNewCuboidCounters(allCuboidsBitSet.length);
+ start = System.currentTimeMillis();
+ long[] valueHashLong = new long[allCuboidsBitSet.length];
+ for (List<String> row : rows) {
+ putRowKeyToHLLNew(row, valueHashLong, cuboidCounters, Hashing.murmur3_128());
+ }
+ totalTime = System.currentTimeMillis() - start;
+ System.out.println("new method cost time : " + totalTime);
+ }
+
+ //test accuracy
+ public void compareAccuracyBasic(final List<List<String>> rows) throws Exception {
+ final long realCardinality = countCardinality(rows);
+ System.out.println("real cardinality : " + realCardinality);
+ //test1
+ long t1 = runAndGetTime(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ HLLCounter counter = new HLLCounter(14, RegisterType.DENSE);
+ final ByteArray[] colHashValues = getNewColHashValues(ROW_LENGTH);
+ HashFunction hf = Hashing.murmur3_32();
+ for (List<String> row : rows) {
+
+ int x = 0;
+ for (String field : row) {
+ Hasher hc = hf.newHasher();
+ colHashValues[x++].set(hc.putString(field).hash().asBytes());
+ }
+
+ Hasher hc = hf.newHasher();
+ for (int position = 0; position < colHashValues.length; position++) {
+ hc.putBytes(colHashValues[position].array());
+ }
+ counter.add(hc.hash().asBytes());
+ }
+ long estimate = counter.getCountEstimate();
+ System.out.println("old method finished. Estimate cardinality : " + estimate + ". Error rate : " + countErrorRate(estimate, realCardinality));
+ }
+ });
+
+
+ long t2 = runAndGetTime(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ HLLCounter counter = new HLLCounter(14, RegisterType.DENSE);
+ HashFunction hf2 = Hashing.murmur3_128();
+ long[] valueHashLong = new long[allCuboidsBitSet.length];
+ for (List<String> row : rows) {
+
+ int x = 0;
+ for (String field : row) {
+ Hasher hc = hf2.newHasher();
+ byte[] bytes = hc.putString(x + field).hash().asBytes();
+ valueHashLong[x++] = Bytes.toLong(bytes);
+ }
+
+ long value = 0;
+ for (int position = 0; position < row.size(); position++) {
+ value += valueHashLong[position];
+ }
+ counter.addHashDirectly(value);
+ }
+ long estimate = counter.getCountEstimate();
+ System.out.println("new method finished. Estimate cardinality : " + estimate + ". Error rate : " + countErrorRate(estimate, realCardinality));
+ }
+ });
+ }
+
+ public void createAllCuboidBitSet() {
+ List<Long> allCuboids = Lists.newArrayList();
+ List<Integer[]> allCuboidsBitSetList = Lists.newArrayList();
+ for (long i = 1; i < baseCuboidId; i++) {
+ allCuboids.add(i);
+ addCuboidBitSet(i, allCuboidsBitSetList);
+ }
+ allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[allCuboidsBitSetList.size()][]);
+ }
+
+ private ByteArray[] getNewColHashValues(int rowLength) {
+ ByteArray[] colHashValues = new ByteArray[rowLength];
+ for (int i = 0; i < rowLength; i++) {
+ colHashValues[i] = new ByteArray();
+ }
+ return colHashValues;
+ }
+
+ private HLLCounter[] getNewCuboidCounters(int cuboidNum) {
+ HLLCounter[] counters = new HLLCounter[cuboidNum];
+ for (int i = 0; i < counters.length; i++)
+ counters[i] = new HLLCounter(14, RegisterType.DENSE);
+ return counters;
+ }
+
+
+ private void addCuboidBitSet(long cuboidId, List<Integer[]> allCuboidsBitSet) {
+ Integer[] indice = new Integer[Long.bitCount(cuboidId)];
+
+ long mask = Long.highestOneBit(baseCuboidId);
+ int position = 0;
+ for (int i = 0; i < ROW_LENGTH; i++) {
+ if ((mask & cuboidId) > 0) {
+ indice[position] = i;
+ position++;
+ }
+ mask = mask >> 1;
+ }
+
+ allCuboidsBitSet.add(indice);
+
+ }
+
+ private long runAndGetTime(TestCase testCase) throws Exception {
+ long start = System.currentTimeMillis();
+ testCase.run();
+ long totalTime = System.currentTimeMillis() - start;
+ return totalTime;
+ }
+
+ interface TestCase {
+ void run() throws Exception;
+ }
+
+ private void putRowKeyToHLL(List<String> row, ByteArray[] colHashValues, HLLCounter[] cuboidCounters, HashFunction hashFunction) {
+ int x = 0;
+ for (String field : row) {
+ Hasher hc = hashFunction.newHasher();
+ colHashValues[x++].set(hc.putString(field).hash().asBytes());
+ }
+
+ for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
+ Hasher hc = hashFunction.newHasher();
+ for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
+ hc.putBytes(colHashValues[allCuboidsBitSet[i][position]].array());
+ //hc.putBytes(seperator);
+ }
+ cuboidCounters[i].add(hc.hash().asBytes());
+ }
+ }
+
+ private void putRowKeyToHLLNew(List<String> row, long[] hashValuesLong, HLLCounter[] cuboidCounters, HashFunction hashFunction) {
+ int x = 0;
+ for (String field : row) {
+ Hasher hc = hashFunction.newHasher();
+ byte[] bytes = hc.putString(x + field).hash().asBytes();
+ hashValuesLong[x++] = Bytes.toLong(bytes);
+ }
+
+ for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
+ long value = 0;
+ for (int position = 0; position < allCuboidsBitSet[i].length; position++) {
+ value += hashValuesLong[allCuboidsBitSet[i][position]];
+ }
+ cuboidCounters[i].addHashDirectly(value);
+ }
+ }
+
+ private List<List<String>> getRandomDataset(int size) {
+ List<List<String>> rows = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ rows.add(getRandomRow());
+ }
+ return rows;
+ }
+
+ private List<List<String>> getSmallCardDataset(int size) {
+ List<List<String>> rows = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ rows.add(getSmallCardRow());
+ }
+ return rows;
+ }
+
+ private List<String> getRandomRow() {
+ List<String> row = new ArrayList<>();
+ for (int i = 0; i < ROW_LENGTH; i++) {
+ row.add(RandomStringUtils.random(10));
+ }
+ return row;
+ }
+
+ private String[] smallCardRow = {"abc", "bcd", "jifea", "feaifj"};
+
+ private Random rand = new Random(System.currentTimeMillis());
+
+ private List<String> getSmallCardRow() {
+ List<String> row = new ArrayList<>();
+ row.add(smallCardRow[rand.nextInt(smallCardRow.length)]);
+ for (int i = 1; i < ROW_LENGTH; i++) {
+ row.add("abc");
+ }
+ return row;
+ }
+
+
+ private int countCardinality(List<List<String>> rows) {
+ Set<String> diffCols = new HashSet<String>();
+ for (List<String> row : rows) {
+ StringBuilder sb = new StringBuilder();
+ for (String str : row) {
+ sb.append(str);
+ }
+ diffCols.add(sb.toString());
+ }
+ return diffCols.size();
+ }
+
+ private double countErrorRate(long estimate, long real) {
+ double rate = Math.abs((estimate - real) * 1.0) / real;
+ return rate;
+ }
+}
[05/35] kylin git commit: minor,
move 1 distinct sql file to h2_uncapable
Posted by li...@apache.org.
minor, move 1 distinct sql file to h2_uncapable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/997d4298
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/997d4298
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/997d4298
Branch: refs/heads/master-hbase0.98
Commit: 997d4298e506b75b93e75d8410f8ab2471975a03
Parents: 9e03c71
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Mar 28 15:59:44 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Tue Mar 28 16:02:45 2017 +0800
----------------------------------------------------------------------
.../resources/query/sql_distinct/query08.sql | 24 --------------------
.../query/sql_h2_uncapable/query05.sql | 24 ++++++++++++++++++++
2 files changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/997d4298/kylin-it/src/test/resources/query/sql_distinct/query08.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct/query08.sql b/kylin-it/src/test/resources/query/sql_distinct/query08.sql
deleted file mode 100644
index 60f02e7..0000000
--- a/kylin-it/src/test/resources/query/sql_distinct/query08.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select cal_dt,
- sum(price) as GMV,
- count(1) as TRANS_CNT,
- count(distinct seller_id, lstg_format_name) as DIST_SELLER_FORMAT
- from test_kylin_fact
- group by cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/997d4298/kylin-it/src/test/resources/query/sql_h2_uncapable/query05.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_h2_uncapable/query05.sql b/kylin-it/src/test/resources/query/sql_h2_uncapable/query05.sql
new file mode 100644
index 0000000..60f02e7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_h2_uncapable/query05.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select cal_dt,
+ sum(price) as GMV,
+ count(1) as TRANS_CNT,
+ count(distinct seller_id, lstg_format_name) as DIST_SELLER_FORMAT
+ from test_kylin_fact
+ group by cal_dt
[22/35] kylin git commit: Merge commit
'1e8b6a5e7b1350930c68e750fb1ce75c27428082'
Posted by li...@apache.org.
Merge commit '1e8b6a5e7b1350930c68e750fb1ce75c27428082'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9250d9ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9250d9ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9250d9ba
Branch: refs/heads/master-hbase0.98
Commit: 9250d9baa6940bf7aa0a76a0025b852a356e2f03
Parents: ee5397b 1e8b6a5
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 31 20:22:16 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 31 20:22:16 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 37 ++++++++++++++------
.../apache/kylin/common/KylinConfigTest.java | 31 +++++++++++++++-
2 files changed, 56 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[16/35] kylin git commit: KYLIN-2521 upgrade calcite to 1.12
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/19d5b3de/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 2c308f5..c199c31 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,25 +17,6 @@
*/
package org.apache.calcite.sql2rel;
-import static org.apache.calcite.sql.SqlUtil.stripAs;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.AbstractList;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
import org.apache.calcite.avatica.util.Spaces;
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.plan.Convention;
@@ -106,6 +88,7 @@ import org.apache.calcite.schema.ModifiableTable;
import org.apache.calcite.schema.ModifiableView;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.Wrapper;
import org.apache.calcite.sql.JoinConditionType;
import org.apache.calcite.sql.JoinType;
import org.apache.calcite.sql.SemiJoinType;
@@ -125,6 +108,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlMatchRecognize;
import org.apache.calcite.sql.SqlMerge;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
@@ -161,6 +145,7 @@ import org.apache.calcite.sql.validate.ListScope;
import org.apache.calcite.sql.validate.ParameterScope;
import org.apache.calcite.sql.validate.SelectScope;
import org.apache.calcite.sql.validate.SqlMonotonicity;
+import org.apache.calcite.sql.validate.SqlNameMatcher;
import org.apache.calcite.sql.validate.SqlQualified;
import org.apache.calcite.sql.validate.SqlUserDefinedTableFunction;
import org.apache.calcite.sql.validate.SqlUserDefinedTableMacro;
@@ -168,6 +153,7 @@ import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql.validate.SqlValidatorImpl;
import org.apache.calcite.sql.validate.SqlValidatorNamespace;
import org.apache.calcite.sql.validate.SqlValidatorScope;
+import org.apache.calcite.sql.validate.SqlValidatorTable;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.ImmutableBitSet;
@@ -178,7 +164,6 @@ import org.apache.calcite.util.NumberUtil;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.trace.CalciteTrace;
-import org.slf4j.Logger;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
@@ -190,6 +175,28 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.util.AbstractList;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.apache.calcite.sql.SqlUtil.stripAs;
+import static org.apache.calcite.util.Static.RESOURCE;
+
/*
* The code has synced with calcite. Hope one day, we could remove the hardcode override point.
* OVERRIDE POINT:
@@ -211,7 +218,7 @@ public class SqlToRelConverter {
//~ Static fields/initializers ---------------------------------------------
protected static final Logger SQL2REL_LOGGER =
- CalciteTrace.getSqlToRelTracer();
+ CalciteTrace.getSqlToRelTracer();
private static final BigDecimal TWO = BigDecimal.valueOf(2L);
@@ -222,7 +229,7 @@ public class SqlToRelConverter {
@Deprecated // to be removed before 2.0
public static final int DEFAULT_IN_SUBQUERY_THRESHOLD =
- DEFAULT_IN_SUB_QUERY_THRESHOLD;
+ DEFAULT_IN_SUB_QUERY_THRESHOLD;
//~ Instance fields --------------------------------------------------------
@@ -230,7 +237,6 @@ public class SqlToRelConverter {
protected final RexBuilder rexBuilder;
protected final Prepare.CatalogReader catalogReader;
protected final RelOptCluster cluster;
- private DefaultValueFactory defaultValueFactory;
private SubQueryConverter subQueryConverter;
protected final List<RelNode> leaves = new ArrayList<>();
private final List<SqlDynamicParam> dynamicParamSqlNodes = new ArrayList<>();
@@ -244,7 +250,7 @@ public class SqlToRelConverter {
* Fields used in name resolution for correlated sub-queries.
*/
private final Map<CorrelationId, DeferredLookup> mapCorrelToDeferred =
- new HashMap<>();
+ new HashMap<>();
/**
* Stack of names of datasets requested by the <code>
@@ -258,7 +264,7 @@ public class SqlToRelConverter {
* already been evaluated.
*/
private final Map<SqlNode, RexNode> mapConvertedNonCorrSubqs =
- new HashMap<>();
+ new HashMap<>();
public final RelOptTable.ViewExpander viewExpander;
@@ -275,44 +281,43 @@ public class SqlToRelConverter {
*/
@Deprecated // to be removed before 2.0
public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptPlanner planner,
- RexBuilder rexBuilder,
- SqlRexConvertletTable convertletTable) {
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptPlanner planner,
+ RexBuilder rexBuilder,
+ SqlRexConvertletTable convertletTable) {
this(viewExpander, validator, catalogReader,
- RelOptCluster.create(planner, rexBuilder), convertletTable,
- Config.DEFAULT);
+ RelOptCluster.create(planner, rexBuilder), convertletTable,
+ Config.DEFAULT);
}
@Deprecated // to be removed before 2.0
public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptCluster cluster,
- SqlRexConvertletTable convertletTable) {
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable) {
this(viewExpander, validator, catalogReader, cluster, convertletTable,
- Config.DEFAULT);
+ Config.DEFAULT);
}
/* Creates a converter. */
public SqlToRelConverter(
- RelOptTable.ViewExpander viewExpander,
- SqlValidator validator,
- Prepare.CatalogReader catalogReader,
- RelOptCluster cluster,
- SqlRexConvertletTable convertletTable,
- Config config) {
+ RelOptTable.ViewExpander viewExpander,
+ SqlValidator validator,
+ Prepare.CatalogReader catalogReader,
+ RelOptCluster cluster,
+ SqlRexConvertletTable convertletTable,
+ Config config) {
this.viewExpander = viewExpander;
this.opTab =
- (validator
- == null) ? SqlStdOperatorTable.instance()
- : validator.getOperatorTable();
+ (validator
+ == null) ? SqlStdOperatorTable.instance()
+ : validator.getOperatorTable();
this.validator = validator;
this.catalogReader = catalogReader;
- this.defaultValueFactory = new NullDefaultValueFactory();
this.subQueryConverter = new NoOpSubQueryConverter();
this.rexBuilder = cluster.getRexBuilder();
this.typeFactory = rexBuilder.getTypeFactory();
@@ -393,21 +398,11 @@ public class SqlToRelConverter {
* @param alreadyConvertedNonCorrSubqs the other map
*/
public void addConvertedNonCorrSubqs(
- Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
+ Map<SqlNode, RexNode> alreadyConvertedNonCorrSubqs) {
mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs);
}
/**
- * Set a new DefaultValueFactory. To have any effect, this must be called
- * before any convert method.
- *
- * @param factory new DefaultValueFactory
- */
- public void setDefaultValueFactory(DefaultValueFactory factory) {
- defaultValueFactory = factory;
- }
-
- /**
* Sets a new SubQueryConverter. To have any effect, this must be called
* before any convert method.
*
@@ -438,36 +433,36 @@ public class SqlToRelConverter {
// validator type information associated with its result,
// hence the namespace check above.)
final List<RelDataTypeField> validatedFields =
- validator.getValidatedNodeType(query).getFieldList();
+ validator.getValidatedNodeType(query).getFieldList();
final RelDataType validatedRowType =
- validator.getTypeFactory().createStructType(
- Pair.right(validatedFields),
- SqlValidatorUtil.uniquify(Pair.left(validatedFields),
- catalogReader.isCaseSensitive()));
+ validator.getTypeFactory().createStructType(
+ Pair.right(validatedFields),
+ SqlValidatorUtil.uniquify(Pair.left(validatedFields),
+ catalogReader.nameMatcher().isCaseSensitive()));
final List<RelDataTypeField> convertedFields =
- result.getRowType().getFieldList().subList(0, validatedFields.size());
+ result.getRowType().getFieldList().subList(0, validatedFields.size());
final RelDataType convertedRowType =
- validator.getTypeFactory().createStructType(convertedFields);
+ validator.getTypeFactory().createStructType(convertedFields);
if (!RelOptUtil.equal("validated row type", validatedRowType,
- "converted row type", convertedRowType, Litmus.IGNORE)) {
+ "converted row type", convertedRowType, Litmus.IGNORE)) {
throw new AssertionError("Conversion to relational algebra failed to "
- + "preserve datatypes:\n"
- + "validated type:\n"
- + validatedRowType.getFullTypeString()
- + "\nconverted type:\n"
- + convertedRowType.getFullTypeString()
- + "\nrel:\n"
- + RelOptUtil.toString(result));
+ + "preserve datatypes:\n"
+ + "validated type:\n"
+ + validatedRowType.getFullTypeString()
+ + "\nconverted type:\n"
+ + convertedRowType.getFullTypeString()
+ + "\nrel:\n"
+ + RelOptUtil.toString(result));
}
}
public RelNode flattenTypes(
- RelNode rootRel,
- boolean restructure) {
+ RelNode rootRel,
+ boolean restructure) {
RelStructuredTypeFlattener typeFlattener =
- new RelStructuredTypeFlattener(rexBuilder, createToRelContext(), restructure);
+ new RelStructuredTypeFlattener(rexBuilder, createToRelContext(), restructure);
return typeFlattener.rewrite(rootRel);
}
@@ -514,20 +509,20 @@ public class SqlToRelConverter {
if (isTrimUnusedFields()) {
final RelFieldTrimmer trimmer = newFieldTrimmer();
final List<RelCollation> collations =
- rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
+ rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE);
rootRel = trimmer.trim(rootRel);
if (!ordered
- && collations != null
- && !collations.isEmpty()
- && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
+ && collations != null
+ && !collations.isEmpty()
+ && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) {
final RelTraitSet traitSet = rootRel.getTraitSet()
- .replace(RelCollationTraitDef.INSTANCE, collations);
+ .replace(RelCollationTraitDef.INSTANCE, collations);
rootRel = rootRel.copy(traitSet, rootRel.getInputs());
}
if (SQL2REL_LOGGER.isDebugEnabled()) {
SQL2REL_LOGGER.debug(
- RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel,
- SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel,
+ SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
}
return rootRel;
@@ -540,7 +535,7 @@ public class SqlToRelConverter {
*/
protected RelFieldTrimmer newFieldTrimmer() {
final RelBuilder relBuilder =
- RelFactories.LOGICAL_BUILDER.create(cluster, null);
+ RelFactories.LOGICAL_BUILDER.create(cluster, null);
return new RelFieldTrimmer(validator, relBuilder);
}
@@ -556,17 +551,18 @@ public class SqlToRelConverter {
* the query will be part of a view.
*/
public RelRoot convertQuery(
- SqlNode query,
- final boolean needsValidation,
- final boolean top) {
- SqlNode origQuery = query; /* OVERRIDE POINT */
+ SqlNode query,
+ final boolean needsValidation,
+ final boolean top) {
+ SqlNode origQuery = query; /* OVERRIDE POINT */
+
if (needsValidation) {
query = validator.validate(query);
}
RelMetadataQuery.THREAD_PROVIDERS.set(
- JaninoRelMetadataProvider.of(cluster.getMetadataProvider()));
+ JaninoRelMetadataProvider.of(cluster.getMetadataProvider()));
RelNode result = convertQueryRecursive(query, top, null).rel;
if (top) {
if (isStream(query)) {
@@ -583,27 +579,28 @@ public class SqlToRelConverter {
if (SQL2REL_LOGGER.isDebugEnabled()) {
SQL2REL_LOGGER.debug(
- RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
- result, SqlExplainFormat.TEXT,
- SqlExplainLevel.EXPPLAN_ATTRIBUTES));
+ RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode",
+ result, SqlExplainFormat.TEXT,
+ SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
final RelDataType validatedRowType = validator.getValidatedNodeType(query);
- RelRoot origResult = RelRoot.of(result, validatedRowType, query.getKind())
- .withCollation(collation);
+ RelRoot origResult = RelRoot.of(result, validatedRowType, query.getKind())
+ .withCollation(collation);
return hackSelectStar(origQuery, origResult);
}
+
/* OVERRIDE POINT */
private RelRoot hackSelectStar(SqlNode query, RelRoot root) {
-// /*
-// * Rel tree is like:
-// *
-// * LogicalSort (optional)
-// * |- LogicalProject
-// * |- LogicalFilter (optional)
-// * |- OLAPTableScan or LogicalJoin
-// */
+ // /*
+ // * Rel tree is like:
+ // *
+ // * LogicalSort (optional)
+ // * |- LogicalProject
+ // * |- LogicalFilter (optional)
+ // * |- OLAPTableScan or LogicalJoin
+ // */
LogicalProject rootPrj = null;
LogicalSort rootSort = null;
if (root.rel instanceof LogicalProject) {
@@ -614,16 +611,16 @@ public class SqlToRelConverter {
} else {
return root;
}
-//
+ //
RelNode input = rootPrj.getInput();
-// if (!(//
-// isAmong(input, "OLAPTableScan", "LogicalJoin")//
-// || (isAmong(input, "LogicalFilter") && isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
-// ))
-// return root;
-//
-// if (rootPrj.getRowType().getFieldCount() < input.getRowType().getFieldCount())
-// return root;
+ // if (!(//
+ // isAmong(input, "OLAPTableScan", "LogicalJoin")//
+ // || (isAmong(input, "LogicalFilter") && isAmong(input.getInput(0), "OLAPTableScan", "LogicalJoin"))//
+ // ))
+ // return root;
+ //
+ // if (rootPrj.getRowType().getFieldCount() < input.getRowType().getFieldCount())
+ // return root;
RelDataType inType = rootPrj.getRowType();
List<String> inFields = inType.getFieldNames();
@@ -654,25 +651,17 @@ public class SqlToRelConverter {
return root;
}
- private boolean isAmong(RelNode rel, String... names) {
- String simpleName = rel.getClass().getSimpleName();
- for (String n : names) {
- if (simpleName.equals(n))
- return true;
- }
- return false;
- }
private static boolean isStream(SqlNode query) {
return query instanceof SqlSelect
- && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
+ && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM);
}
public static boolean isOrdered(SqlNode query) {
switch (query.getKind()) {
case SELECT:
return ((SqlSelect) query).getOrderList() != null
- && ((SqlSelect) query).getOrderList().size() > 0;
+ && ((SqlSelect) query).getOrderList().size() > 0;
case WITH:
return isOrdered(((SqlWith) query).body);
case ORDER_BY:
@@ -709,7 +698,7 @@ public class SqlToRelConverter {
* Factory method for creating translation workspace.
*/
protected Blackboard createBlackboard(SqlValidatorScope scope,
- Map<String, RexNode> nameToNodeMap, boolean top) {
+ Map<String, RexNode> nameToNodeMap, boolean top) {
return new Blackboard(scope, nameToNodeMap, top);
}
@@ -718,44 +707,44 @@ public class SqlToRelConverter {
* derived class may override.
*/
protected void convertSelectImpl(
- final Blackboard bb,
- SqlSelect select) {
+ final Blackboard bb,
+ SqlSelect select) {
convertFrom(
- bb,
- select.getFrom());
+ bb,
+ select.getFrom());
convertWhere(
- bb,
- select.getWhere());
+ bb,
+ select.getWhere());
final List<SqlNode> orderExprList = new ArrayList<>();
final List<RelFieldCollation> collationList = new ArrayList<>();
gatherOrderExprs(
- bb,
- select,
- select.getOrderList(),
- orderExprList,
- collationList);
+ bb,
+ select,
+ select.getOrderList(),
+ orderExprList,
+ collationList);
final RelCollation collation =
- cluster.traitSet().canonize(RelCollations.of(collationList));
+ cluster.traitSet().canonize(RelCollations.of(collationList));
if (validator.isAggregate(select)) {
convertAgg(
- bb,
- select,
- orderExprList);
+ bb,
+ select,
+ orderExprList);
} else {
convertSelectList(
- bb,
- select,
- orderExprList);
+ bb,
+ select,
+ orderExprList);
}
if (select.isDistinct()) {
distinctify(bb, true);
}
convertOrder(
- select, bb, collation, orderExprList, select.getOffset(),
- select.getFetch());
+ select, bb, collation, orderExprList, select.getOffset(),
+ select.getFetch());
bb.setRoot(bb.root, true);
}
@@ -772,8 +761,8 @@ public class SqlToRelConverter {
* @param checkForDupExprs Check for duplicate expressions
*/
private void distinctify(
- Blackboard bb,
- boolean checkForDupExprs) {
+ Blackboard bb,
+ boolean checkForDupExprs) {
// Look for duplicate expressions in the project.
// Say we have 'select x, y, x, z'.
// Then dups will be {[2, 0]}
@@ -808,8 +797,8 @@ public class SqlToRelConverter {
}
}
rel =
- LogicalProject.create(rel, Pair.left(newProjects),
- Pair.right(newProjects));
+ LogicalProject.create(rel, Pair.left(newProjects),
+ Pair.right(newProjects));
bb.root = rel;
distinctify(bb, false);
rel = bb.root;
@@ -821,18 +810,18 @@ public class SqlToRelConverter {
final int origin = origins.get(i);
RelDataTypeField field = fields.get(i);
undoProjects.add(
- Pair.of(
- (RexNode) new RexInputRef(
- squished.get(origin), field.getType()),
- field.getName()));
+ Pair.of(
+ (RexNode) new RexInputRef(
+ squished.get(origin), field.getType()),
+ field.getName()));
}
rel =
- LogicalProject.create(rel, Pair.left(undoProjects),
- Pair.right(undoProjects));
+ LogicalProject.create(rel, Pair.left(undoProjects),
+ Pair.right(undoProjects));
bb.setRoot(
- rel,
- false);
+ rel,
+ false);
return;
}
@@ -840,14 +829,14 @@ public class SqlToRelConverter {
// Usual case: all of the expressions in the SELECT clause are
// different.
final ImmutableBitSet groupSet =
- ImmutableBitSet.range(rel.getRowType().getFieldCount());
+ ImmutableBitSet.range(rel.getRowType().getFieldCount());
rel =
- createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
- ImmutableList.<AggregateCall>of());
+ createAggregate(bb, false, groupSet, ImmutableList.of(groupSet),
+ ImmutableList.<AggregateCall>of());
bb.setRoot(
- rel,
- false);
+ rel,
+ false);
}
private int findExpr(RexNode seek, List<RexNode> exprs, int count) {
@@ -873,28 +862,28 @@ public class SqlToRelConverter {
* @param fetch Expression for number of rows to fetch
*/
protected void convertOrder(
- SqlSelect select,
- Blackboard bb,
- RelCollation collation,
- List<SqlNode> orderExprList,
- SqlNode offset,
- SqlNode fetch) {
+ SqlSelect select,
+ Blackboard bb,
+ RelCollation collation,
+ List<SqlNode> orderExprList,
+ SqlNode offset,
+ SqlNode fetch) {
if (select.getOrderList() == null
- || select.getOrderList().getList().isEmpty()) {
+ || select.getOrderList().getList().isEmpty()) {
assert collation.getFieldCollations().isEmpty();
if ((offset == null
- || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
- && fetch == null) {
+ || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO))
+ && fetch == null) {
return;
}
}
// Create a sorter using the previously constructed collations.
bb.setRoot(
- LogicalSort.create(bb.root, collation,
- offset == null ? null : convertExpression(offset),
- fetch == null ? null : convertExpression(fetch)),
- false);
+ LogicalSort.create(bb.root, collation,
+ offset == null ? null : convertExpression(offset),
+ fetch == null ? null : convertExpression(fetch)),
+ false);
// If extra expressions were added to the project list for sorting,
// add another project to remove them. But make the collation empty, because
@@ -905,14 +894,14 @@ public class SqlToRelConverter {
final List<RexNode> exprs = new ArrayList<>();
final RelDataType rowType = bb.root.getRowType();
final int fieldCount =
- rowType.getFieldCount() - orderExprList.size();
+ rowType.getFieldCount() - orderExprList.size();
for (int i = 0; i < fieldCount; i++) {
exprs.add(rexBuilder.makeInputRef(bb.root, i));
}
bb.setRoot(
- LogicalProject.create(bb.root, exprs,
- rowType.getFieldNames().subList(0, fieldCount)),
- false);
+ LogicalProject.create(bb.root, exprs,
+ rowType.getFieldNames().subList(0, fieldCount)),
+ false);
}
}
@@ -922,17 +911,17 @@ public class SqlToRelConverter {
* @param node a RexNode tree
*/
private static boolean containsInOperator(
- SqlNode node) {
+ SqlNode node) {
try {
SqlVisitor<Void> visitor =
- new SqlBasicVisitor<Void>() {
- public Void visit(SqlCall call) {
- if (call.getOperator() instanceof SqlInOperator) {
- throw new Util.FoundOne(call);
- }
- return super.visit(call);
+ new SqlBasicVisitor<Void>() {
+ public Void visit(SqlCall call) {
+ if (call.getOperator() instanceof SqlInOperator) {
+ throw new Util.FoundOne(call);
}
- };
+ return super.visit(call);
+ }
+ };
node.accept(visitor);
return false;
} catch (Util.FoundOne e) {
@@ -949,11 +938,11 @@ public class SqlToRelConverter {
* @return the transformed SqlNode representation with NOT pushed down.
*/
private static SqlNode pushDownNotForIn(SqlValidatorScope scope,
- SqlNode sqlNode) {
+ SqlNode sqlNode) {
if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) {
SqlCall sqlCall = (SqlCall) sqlNode;
if ((sqlCall.getOperator() == SqlStdOperatorTable.AND)
- || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
+ || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) {
SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands;
for (int i = 0; i < sqlOperands.length; i++) {
sqlOperands[i] = pushDownNotForIn(scope, sqlOperands[i]);
@@ -967,22 +956,30 @@ public class SqlToRelConverter {
SqlNode[] andOperands = childSqlCall.getOperands();
SqlNode[] orOperands = new SqlNode[andOperands.length];
for (int i = 0; i < orOperands.length; i++) {
- orOperands[i] = reg(scope, SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]));
+ orOperands[i] = reg(scope,
+ SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO,
+ andOperands[i]));
}
for (int i = 0; i < orOperands.length; i++) {
orOperands[i] = pushDownNotForIn(scope, orOperands[i]);
}
- return reg(scope, SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]));
+ return reg(scope,
+ SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO,
+ orOperands[0], orOperands[1]));
} else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) {
SqlNode[] orOperands = childSqlCall.getOperands();
SqlNode[] andOperands = new SqlNode[orOperands.length];
for (int i = 0; i < andOperands.length; i++) {
- andOperands[i] = reg(scope, SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]));
+ andOperands[i] = reg(scope,
+ SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO,
+ orOperands[i]));
}
for (int i = 0; i < andOperands.length; i++) {
andOperands[i] = pushDownNotForIn(scope, andOperands[i]);
}
- return reg(scope, SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]));
+ return reg(scope,
+ SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO,
+ andOperands[0], andOperands[1]));
} else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) {
SqlNode[] notOperands = childSqlCall.getOperands();
assert notOperands.length == 1;
@@ -990,11 +987,15 @@ public class SqlToRelConverter {
} else if (childSqlCall.getOperator() instanceof SqlInOperator) {
SqlNode[] inOperands = childSqlCall.getOperands();
SqlInOperator inOp =
- (SqlInOperator) childSqlCall.getOperator();
+ (SqlInOperator) childSqlCall.getOperator();
if (inOp.isNotIn()) {
- return reg(scope, SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]));
+ return reg(scope,
+ SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO,
+ inOperands[0], inOperands[1]));
} else {
- return reg(scope, SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]));
+ return reg(scope,
+ SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO,
+ inOperands[0], inOperands[1]));
}
} else {
// childSqlCall is "leaf" node in a logical expression tree
@@ -1013,7 +1014,7 @@ public class SqlToRelConverter {
}
/** Registers with the validator a {@link SqlNode} that has been created
- * during the Sql-to-Rel process. */
+ * during the Sql-to-Rel process. */
private static SqlNode reg(SqlValidatorScope scope, SqlNode e) {
scope.getValidator().deriveType(scope, e);
return e;
@@ -1026,8 +1027,8 @@ public class SqlToRelConverter {
* @param where WHERE clause, may be null
*/
private void convertWhere(
- final Blackboard bb,
- final SqlNode where) {
+ final Blackboard bb,
+ final SqlNode where) {
if (where == null) {
return;
}
@@ -1041,7 +1042,7 @@ public class SqlToRelConverter {
}
final RelFactories.FilterFactory factory =
- RelFactories.DEFAULT_FILTER_FACTORY;
+ RelFactories.DEFAULT_FILTER_FACTORY;
final RelNode filter = factory.createFilter(bb.root, convertedWhere);
final RelNode r;
final CorrelationUse p = getCorrelationUse(bb, filter);
@@ -1049,7 +1050,7 @@ public class SqlToRelConverter {
assert p.r instanceof Filter;
Filter f = (Filter) p.r;
r = LogicalFilter.create(f.getInput(), f.getCondition(),
- ImmutableSet.of(p.id));
+ ImmutableSet.of(p.id));
} else {
r = filter;
}
@@ -1058,9 +1059,9 @@ public class SqlToRelConverter {
}
private void replaceSubQueries(
- final Blackboard bb,
- final SqlNode expr,
- RelOptUtil.Logic logic) {
+ final Blackboard bb,
+ final SqlNode expr,
+ RelOptUtil.Logic logic) {
findSubQueries(bb, expr, logic, false);
for (SubQuery node : bb.subQueryList) {
substituteSubQuery(bb, node);
@@ -1114,14 +1115,14 @@ public class SqlToRelConverter {
if (query instanceof SqlNodeList) {
SqlNodeList valueList = (SqlNodeList) query;
if (!containsNullLiteral(valueList)
- && valueList.size() < config.getInSubQueryThreshold()) {
+ && valueList.size() < config.getInSubQueryThreshold()) {
// We're under the threshold, so convert to OR.
subQuery.expr =
- convertInToOr(
- bb,
- leftKeys,
- valueList,
- notIn);
+ convertInToOr(
+ bb,
+ leftKeys,
+ valueList,
+ notIn);
return;
}
@@ -1153,36 +1154,36 @@ public class SqlToRelConverter {
// and q.indicator <> TRUE"
//
final RelDataType targetRowType =
- SqlTypeUtil.promoteToRowType(typeFactory,
- validator.getValidatedNodeType(leftKeyNode), null);
+ SqlTypeUtil.promoteToRowType(typeFactory,
+ validator.getValidatedNodeType(leftKeyNode), null);
converted =
- convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic,
- notIn, targetRowType);
+ convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic,
+ notIn, targetRowType);
if (converted.indicator) {
// Generate
// emp CROSS JOIN (SELECT COUNT(*) AS c,
// COUNT(deptno) AS ck FROM dept)
final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
final RelNode seek = converted.r.getInput(0); // fragile
final int keyCount = leftKeys.size();
final List<Integer> args = ImmutableIntList.range(0, keyCount);
LogicalAggregate aggregate =
- LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
- ImmutableList.of(
- AggregateCall.create(SqlStdOperatorTable.COUNT, false,
- ImmutableList.<Integer>of(), -1, longType, null),
- AggregateCall.create(SqlStdOperatorTable.COUNT, false,
- args, -1, longType, null)));
+ LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null,
+ ImmutableList.of(
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ ImmutableList.<Integer>of(), -1, longType, null),
+ AggregateCall.create(SqlStdOperatorTable.COUNT, false,
+ args, -1, longType, null)));
LogicalJoin join =
- LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
- ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
+ LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true),
+ ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
bb.setRoot(join, false);
}
final RexNode rex =
- bb.register(converted.r,
- converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
- leftKeys);
+ bb.register(converted.r,
+ converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER,
+ leftKeys);
RelOptUtil.Logic logic = subQuery.logic;
switch (logic) {
@@ -1195,7 +1196,7 @@ public class SqlToRelConverter {
subQuery.expr = translateIn(logic, bb.root, rex);
if (notIn) {
subQuery.expr =
- rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
+ rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
}
return;
@@ -1215,7 +1216,7 @@ public class SqlToRelConverter {
return;
}
converted = convertExists(query, RelOptUtil.SubQueryType.EXISTS,
- subQuery.logic, true, null);
+ subQuery.logic, true, null);
assert !converted.indicator;
if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
return;
@@ -1232,7 +1233,7 @@ public class SqlToRelConverter {
call = (SqlBasicCall) subQuery.node;
query = call.operand(0);
converted = convertExists(query, RelOptUtil.SubQueryType.SCALAR,
- subQuery.logic, true, null);
+ subQuery.logic, true, null);
assert !converted.indicator;
if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
return;
@@ -1247,18 +1248,19 @@ public class SqlToRelConverter {
// select * from unnest(select multiset[deptno] from emps);
//
converted = convertExists(subQuery.node, RelOptUtil.SubQueryType.SCALAR,
- subQuery.logic, true, null);
+ subQuery.logic, true, null);
assert !converted.indicator;
subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
return;
default:
- throw Util.newInternal("unexpected kind of sub-query :" + subQuery.node);
+ throw new AssertionError("unexpected kind of sub-query: "
+ + subQuery.node);
}
}
private RexNode translateIn(RelOptUtil.Logic logic, RelNode root,
- final RexNode rex) {
+ final RexNode rex) {
switch (logic) {
case TRUE:
return rexBuilder.makeLiteral(true);
@@ -1281,12 +1283,12 @@ public class SqlToRelConverter {
final int k = (fieldCount - 1) / 2;
for (int i = 0; i < k; i++) {
rexNode =
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.AND,
+ rexNode,
rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- rexNode,
- rexBuilder.makeCall(
- SqlStdOperatorTable.IS_NOT_NULL,
- rexBuilder.makeFieldAccess(rex, i)));
+ SqlStdOperatorTable.IS_NOT_NULL,
+ rexBuilder.makeFieldAccess(rex, i)));
}
return rexNode;
@@ -1309,33 +1311,33 @@ public class SqlToRelConverter {
final RelNode leftLeft = ((Join) left.getInput()).getLeft();
final int leftLeftCount = leftLeft.getRowType().getFieldCount();
final RelDataType longType =
- typeFactory.createSqlType(SqlTypeName.BIGINT);
+ typeFactory.createSqlType(SqlTypeName.BIGINT);
final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount);
final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1);
final RexNode iRef =
- rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
+ rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1);
final RexLiteral zero =
- rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
+ rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType);
final RexLiteral trueLiteral = rexBuilder.makeLiteral(true);
final RexLiteral falseLiteral = rexBuilder.makeLiteral(false);
final RexNode unknownLiteral =
- rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN);
+ rexBuilder.makeNullLiteral(trueLiteral.getType());
final ImmutableList.Builder<RexNode> args = ImmutableList.builder();
args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero),
- falseLiteral,
- rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
- trueLiteral);
+ falseLiteral,
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef),
+ trueLiteral);
final JoinInfo joinInfo = join.analyzeCondition();
for (int leftKey : joinInfo.leftKeys) {
final RexNode kRef = rexBuilder.makeInputRef(root, leftKey);
args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef),
- unknownLiteral);
+ unknownLiteral);
}
args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef),
- unknownLiteral,
- falseLiteral);
+ unknownLiteral,
+ falseLiteral);
return rexBuilder.makeCall(SqlStdOperatorTable.CASE, args.build());
@@ -1367,24 +1369,24 @@ public class SqlToRelConverter {
* @return Whether the sub-query can be converted to a constant
*/
private boolean convertNonCorrelatedSubQuery(
- SubQuery subQuery,
- Blackboard bb,
- RelNode converted,
- boolean isExists) {
+ SubQuery subQuery,
+ Blackboard bb,
+ RelNode converted,
+ boolean isExists) {
SqlCall call = (SqlBasicCall) subQuery.node;
if (subQueryConverter.canConvertSubQuery()
- && isSubQueryNonCorrelated(converted, bb)) {
+ && isSubQueryNonCorrelated(converted, bb)) {
// First check if the sub-query has already been converted
// because it's a nested sub-query. If so, don't re-evaluate
// it again.
RexNode constExpr = mapConvertedNonCorrSubqs.get(call);
if (constExpr == null) {
constExpr =
- subQueryConverter.convertSubQuery(
- call,
- this,
- isExists,
- config.isExplain());
+ subQueryConverter.convertSubQuery(
+ call,
+ this,
+ isExists,
+ config.isExplain());
}
if (constExpr != null) {
subQuery.expr = constExpr;
@@ -1404,8 +1406,8 @@ public class SqlToRelConverter {
* @return the converted RelNode tree
*/
public RelNode convertToSingleValueSubq(
- SqlNode query,
- RelNode plan) {
+ SqlNode query,
+ RelNode plan) {
// Check whether query is guaranteed to produce a single value.
if (query instanceof SqlSelect) {
SqlSelect select = (SqlSelect) query;
@@ -1413,7 +1415,7 @@ public class SqlToRelConverter {
SqlNodeList groupList = select.getGroup();
if ((selectList.size() == 1)
- && ((groupList == null) || (groupList.size() == 0))) {
+ && ((groupList == null) || (groupList.size() == 0))) {
SqlNode selectExpr = selectList.get(0);
if (selectExpr instanceof SqlCall) {
SqlCall selectExprCall = (SqlCall) selectExpr;
@@ -1425,7 +1427,7 @@ public class SqlToRelConverter {
// If there is a limit with 0 or 1,
// it is ensured to produce a single value
if (select.getFetch() != null
- && select.getFetch() instanceof SqlNumericLiteral) {
+ && select.getFetch() instanceof SqlNumericLiteral) {
SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch();
if (((BigDecimal) limitNum.getValue()).intValue() < 2) {
return plan;
@@ -1438,16 +1440,16 @@ public class SqlToRelConverter {
// whether SingleValueAgg is necessary
SqlCall exprCall = (SqlCall) query;
if (exprCall.getOperator()
- instanceof SqlValuesOperator
- && Util.isSingleValue(exprCall)) {
+ instanceof SqlValuesOperator
+ && Util.isSingleValue(exprCall)) {
return plan;
}
}
// If not, project SingleValueAgg
return RelOptUtil.createSingleValueAggRel(
- cluster,
- plan);
+ cluster,
+ plan);
}
/**
@@ -1459,51 +1461,51 @@ public class SqlToRelConverter {
* @return converted expression
*/
private RexNode convertInToOr(
- final Blackboard bb,
- final List<RexNode> leftKeys,
- SqlNodeList valuesList,
- boolean isNotIn) {
+ final Blackboard bb,
+ final List<RexNode> leftKeys,
+ SqlNodeList valuesList,
+ boolean isNotIn) {
final List<RexNode> comparisons = new ArrayList<>();
for (SqlNode rightVals : valuesList) {
RexNode rexComparison;
if (leftKeys.size() == 1) {
rexComparison =
- rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- leftKeys.get(0),
- ensureSqlType(leftKeys.get(0).getType(),
- bb.convertExpression(rightVals)));
+ rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ leftKeys.get(0),
+ ensureSqlType(leftKeys.get(0).getType(),
+ bb.convertExpression(rightVals)));
} else {
assert rightVals instanceof SqlCall;
final SqlBasicCall call = (SqlBasicCall) rightVals;
assert (call.getOperator() instanceof SqlRowOperator)
- && call.operandCount() == leftKeys.size();
+ && call.operandCount() == leftKeys.size();
rexComparison =
- RexUtil.composeConjunction(
- rexBuilder,
- Iterables.transform(
- Pair.zip(leftKeys, call.getOperandList()),
- new Function<Pair<RexNode, SqlNode>, RexNode>() {
- public RexNode apply(Pair<RexNode, SqlNode> pair) {
- return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
- pair.left,
- ensureSqlType(pair.left.getType(),
- bb.convertExpression(pair.right)));
- }
- }),
- false);
+ RexUtil.composeConjunction(
+ rexBuilder,
+ Iterables.transform(
+ Pair.zip(leftKeys, call.getOperandList()),
+ new Function<Pair<RexNode, SqlNode>, RexNode>() {
+ public RexNode apply(Pair<RexNode, SqlNode> pair) {
+ return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS,
+ pair.left,
+ ensureSqlType(pair.left.getType(),
+ bb.convertExpression(pair.right)));
+ }
+ }),
+ false);
}
comparisons.add(rexComparison);
}
RexNode result =
- RexUtil.composeDisjunction(rexBuilder, comparisons, true);
+ RexUtil.composeDisjunction(rexBuilder, comparisons, true);
assert result != null;
if (isNotIn) {
result =
- rexBuilder.makeCall(
- SqlStdOperatorTable.NOT,
- result);
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.NOT,
+ result);
}
return result;
@@ -1514,8 +1516,8 @@ public class SqlToRelConverter {
* returns the expression unchanged. */
private RexNode ensureSqlType(RelDataType type, RexNode node) {
if (type.getSqlTypeName() == node.getType().getSqlTypeName()
- || (type.getSqlTypeName() == SqlTypeName.VARCHAR
- && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
+ || (type.getSqlTypeName() == SqlTypeName.VARCHAR
+ && node.getType().getSqlTypeName() == SqlTypeName.CHAR)) {
return node;
}
return rexBuilder.ensureType(type, node, true);
@@ -1551,18 +1553,17 @@ public class SqlToRelConverter {
* approximation (say representing UNKNOWN as FALSE)
* @param notIn Whether the operation is NOT IN
* @return join expression
- * @pre extraExpr == null || extraName != null
*/
private RelOptUtil.Exists convertExists(
- SqlNode seek,
- RelOptUtil.SubQueryType subQueryType,
- RelOptUtil.Logic logic,
- boolean notIn,
- RelDataType targetDataType) {
+ SqlNode seek,
+ RelOptUtil.SubQueryType subQueryType,
+ RelOptUtil.Logic logic,
+ boolean notIn,
+ RelDataType targetDataType) {
final SqlValidatorScope seekScope =
- (seek instanceof SqlSelect)
- ? validator.getSelectScope((SqlSelect) seek)
- : null;
+ (seek instanceof SqlSelect)
+ ? validator.getSelectScope((SqlSelect) seek)
+ : null;
final Blackboard seekBb = createBlackboard(seekScope, null, false);
RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType);
@@ -1570,9 +1571,9 @@ public class SqlToRelConverter {
}
private RelNode convertQueryOrInList(
- Blackboard bb,
- SqlNode seek,
- RelDataType targetRowType) {
+ Blackboard bb,
+ SqlNode seek,
+ RelDataType targetRowType) {
// NOTE: Once we start accepting single-row queries as row constructors,
// there will be an ambiguity here for a case like X IN ((SELECT Y FROM
// Z)). The SQL standard resolves the ambiguity by saying that a lone
@@ -1581,22 +1582,22 @@ public class SqlToRelConverter {
// return multiple rows.
if (seek instanceof SqlNodeList) {
return convertRowValues(
- bb,
- seek,
- ((SqlNodeList) seek).getList(),
- false,
- targetRowType);
+ bb,
+ seek,
+ ((SqlNodeList) seek).getList(),
+ false,
+ targetRowType);
} else {
return convertQueryRecursive(seek, false, null).project();
}
}
private RelNode convertRowValues(
- Blackboard bb,
- SqlNode rowList,
- Collection<SqlNode> rows,
- boolean allowLiteralsOnly,
- RelDataType targetRowType) {
+ Blackboard bb,
+ SqlNode rowList,
+ Collection<SqlNode> rows,
+ boolean allowLiteralsOnly,
+ RelDataType targetRowType) {
// NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of
// literals into a single LogicalValues; this gives the optimizer a smaller
// input tree. For everything else (computed expressions, row
@@ -1604,16 +1605,16 @@ public class SqlToRelConverter {
// LogicalOneRow.
final ImmutableList.Builder<ImmutableList<RexLiteral>> tupleList =
- ImmutableList.builder();
+ ImmutableList.builder();
final RelDataType rowType;
if (targetRowType != null) {
rowType = targetRowType;
} else {
rowType =
- SqlTypeUtil.promoteToRowType(
- typeFactory,
- validator.getValidatedNodeType(rowList),
- null);
+ SqlTypeUtil.promoteToRowType(
+ typeFactory,
+ validator.getValidatedNodeType(rowList),
+ null);
}
final List<RelNode> unionInputs = new ArrayList<>();
@@ -1624,11 +1625,11 @@ public class SqlToRelConverter {
ImmutableList.Builder<RexLiteral> tuple = ImmutableList.builder();
for (Ord<SqlNode> operand : Ord.zip(call.operands)) {
RexLiteral rexLiteral =
- convertLiteralInValuesList(
- operand.e,
- bb,
- rowType,
- operand.i);
+ convertLiteralInValuesList(
+ operand.e,
+ bb,
+ rowType,
+ operand.i);
if ((rexLiteral == null) && allowLiteralsOnly) {
return null;
}
@@ -1645,11 +1646,11 @@ public class SqlToRelConverter {
}
} else {
RexLiteral rexLiteral =
- convertLiteralInValuesList(
- node,
- bb,
- rowType,
- 0);
+ convertLiteralInValuesList(
+ node,
+ bb,
+ rowType,
+ 0);
if ((rexLiteral != null) && config.isCreateValuesRel()) {
tupleList.add(ImmutableList.of(rexLiteral));
continue;
@@ -1661,14 +1662,14 @@ public class SqlToRelConverter {
// convert "1" to "row(1)"
call =
- (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
- SqlParserPos.ZERO,
- node);
+ (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(
+ SqlParserPos.ZERO,
+ node);
}
unionInputs.add(convertRowConstructor(bb, call));
}
LogicalValues values =
- LogicalValues.create(cluster, rowType, tupleList.build());
+ LogicalValues.create(cluster, rowType, tupleList.build());
RelNode resultRel;
if (unionInputs.isEmpty()) {
resultRel = values;
@@ -1683,10 +1684,10 @@ public class SqlToRelConverter {
}
private RexLiteral convertLiteralInValuesList(
- SqlNode sqlNode,
- Blackboard bb,
- RelDataType rowType,
- int iField) {
+ SqlNode sqlNode,
+ Blackboard bb,
+ RelDataType rowType,
+ int iField) {
if (!(sqlNode instanceof SqlLiteral)) {
return null;
}
@@ -1700,9 +1701,9 @@ public class SqlToRelConverter {
}
RexNode literalExpr =
- exprConverter.convertLiteral(
- bb,
- (SqlLiteral) sqlNode);
+ exprConverter.convertLiteral(
+ bb,
+ (SqlLiteral) sqlNode);
if (!(literalExpr instanceof RexLiteral)) {
assert literalExpr.isA(SqlKind.CAST);
@@ -1720,23 +1721,23 @@ public class SqlToRelConverter {
if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) {
BigDecimal roundedValue =
- NumberUtil.rescaleBigDecimal(
- (BigDecimal) value,
- type.getScale());
+ NumberUtil.rescaleBigDecimal(
+ (BigDecimal) value,
+ type.getScale());
return rexBuilder.makeExactLiteral(
- roundedValue,
- type);
+ roundedValue,
+ type);
}
if ((value instanceof NlsString)
- && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
+ && (type.getSqlTypeName() == SqlTypeName.CHAR)) {
// pad fixed character type
NlsString unpadded = (NlsString) value;
return rexBuilder.makeCharLiteral(
- new NlsString(
- Spaces.padRight(unpadded.getValue(), type.getPrecision()),
- unpadded.getCharsetName(),
- unpadded.getCollation()));
+ new NlsString(
+ Spaces.padRight(unpadded.getValue(), type.getPrecision()),
+ unpadded.getCharsetName(),
+ unpadded.getCollation()));
}
return literal;
}
@@ -1764,10 +1765,10 @@ public class SqlToRelConverter {
* sub-query
*/
private void findSubQueries(
- Blackboard bb,
- SqlNode node,
- RelOptUtil.Logic logic,
- boolean registerOnlyScalarSubQueries) {
+ Blackboard bb,
+ SqlNode node,
+ RelOptUtil.Logic logic,
+ boolean registerOnlyScalarSubQueries) {
final SqlKind kind = node.getKind();
switch (kind) {
case EXISTS:
@@ -1778,7 +1779,7 @@ public class SqlToRelConverter {
case CURSOR:
case SCALAR_QUERY:
if (!registerOnlyScalarSubQueries
- || (kind == SqlKind.SCALAR_QUERY)) {
+ || (kind == SqlKind.SCALAR_QUERY)) {
bb.registerSubQuery(node, RelOptUtil.Logic.TRUE_FALSE);
}
return;
@@ -1797,19 +1798,19 @@ public class SqlToRelConverter {
// In the case of an IN expression, locate scalar
// sub-queries so we can convert them to constants
findSubQueries(
- bb,
- operand,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubQueries);
+ bb,
+ operand,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubQueries);
}
}
} else if (node instanceof SqlNodeList) {
for (SqlNode child : (SqlNodeList) node) {
findSubQueries(
- bb,
- child,
- logic,
- kind == SqlKind.IN || registerOnlyScalarSubQueries);
+ bb,
+ child,
+ logic,
+ kind == SqlKind.IN || registerOnlyScalarSubQueries);
}
}
@@ -1840,10 +1841,10 @@ public class SqlToRelConverter {
* @return Converted expression
*/
public RexNode convertExpression(
- SqlNode node) {
+ SqlNode node) {
Map<String, RelDataType> nameToTypeMap = Collections.emptyMap();
final ParameterScope scope =
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
final Blackboard bb = createBlackboard(scope, null, false);
return bb.convertExpression(node);
}
@@ -1860,14 +1861,14 @@ public class SqlToRelConverter {
* @return Converted expression
*/
public RexNode convertExpression(
- SqlNode node,
- Map<String, RexNode> nameToNodeMap) {
+ SqlNode node,
+ Map<String, RexNode> nameToNodeMap) {
final Map<String, RelDataType> nameToTypeMap = new HashMap<>();
for (Map.Entry<String, RexNode> entry : nameToNodeMap.entrySet()) {
nameToTypeMap.put(entry.getKey(), entry.getValue().getType());
}
final ParameterScope scope =
- new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
+ new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap);
final Blackboard bb = createBlackboard(scope, nameToNodeMap, false);
return bb.convertExpression(node);
}
@@ -1884,8 +1885,8 @@ public class SqlToRelConverter {
* @return null to proceed with the usual expression translation process
*/
protected RexNode convertExtendedExpression(
- SqlNode node,
- Blackboard bb) {
+ SqlNode node,
+ Blackboard bb) {
return null;
}
@@ -1894,7 +1895,7 @@ public class SqlToRelConverter {
SqlCall aggCall = call.operand(0);
SqlNode windowOrRef = call.operand(1);
final SqlWindow window =
- validator.resolveWindow(windowOrRef, bb.scope, true);
+ validator.resolveWindow(windowOrRef, bb.scope, true);
// ROW_NUMBER() expects specific kind of framing.
if (aggCall.getKind() == SqlKind.ROW_NUMBER) {
@@ -1904,7 +1905,7 @@ public class SqlToRelConverter {
}
final SqlNodeList partitionList = window.getPartitionList();
final ImmutableList.Builder<RexNode> partitionKeys =
- ImmutableList.builder();
+ ImmutableList.builder();
for (SqlNode partition : partitionList) {
partitionKeys.add(bb.convertExpression(partition));
}
@@ -1918,11 +1919,11 @@ public class SqlToRelConverter {
orderList = bb.scope.getOrderList();
if (orderList == null) {
throw new AssertionError(
- "Relation should have sort key for implicit ORDER BY");
+ "Relation should have sort key for implicit ORDER BY");
}
}
final ImmutableList.Builder<RexFieldCollation> orderKeys =
- ImmutableList.builder();
+ ImmutableList.builder();
final Set<SqlKind> flags = EnumSet.noneOf(SqlKind.class);
for (SqlNode order : orderList) {
flags.clear();
@@ -1930,22 +1931,23 @@ public class SqlToRelConverter {
orderKeys.add(new RexFieldCollation(e, flags));
}
try {
- Util.permAssert(bb.window == null, "already in window agg mode");
+ Preconditions.checkArgument(bb.window == null,
+ "already in window agg mode");
bb.window = window;
RexNode rexAgg = exprConverter.convertCall(bb, aggCall);
rexAgg =
- rexBuilder.ensureType(
- validator.getValidatedNodeType(call), rexAgg, false);
+ rexBuilder.ensureType(
+ validator.getValidatedNodeType(call), rexAgg, false);
// Walk over the tree and apply 'over' to all agg functions. This is
// necessary because the returned expression is not necessarily a call
// to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x).
final RexShuttle visitor =
- new HistogramShuttle(
- partitionKeys.build(), orderKeys.build(),
- RexWindowBound.create(window.getLowerBound(), lowerBound),
- RexWindowBound.create(window.getUpperBound(), upperBound),
- window);
+ new HistogramShuttle(
+ partitionKeys.build(), orderKeys.build(),
+ RexWindowBound.create(window.getLowerBound(), lowerBound),
+ RexWindowBound.create(window.getUpperBound(), upperBound),
+ window);
return rexAgg.accept(visitor);
} finally {
bb.window = null;
@@ -1970,8 +1972,8 @@ public class SqlToRelConverter {
* </ul>
*/
protected void convertFrom(
- Blackboard bb,
- SqlNode from) {
+ Blackboard bb,
+ SqlNode from) {
if (from == null) {
bb.setRoot(LogicalValues.createOneRow(cluster), false);
return;
@@ -1980,6 +1982,10 @@ public class SqlToRelConverter {
final SqlCall call;
final SqlNode[] operands;
switch (from.getKind()) {
+ case MATCH_RECOGNIZE:
+ convertMatchRecognize(bb, (SqlCall) from);
+ return;
+
case AS:
convertFrom(bb, ((SqlCall) from).operand(0));
return;
@@ -1997,54 +2003,36 @@ public class SqlToRelConverter {
SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]);
if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) {
String sampleName =
- ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
- .getName();
+ ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec)
+ .getName();
datasetStack.push(sampleName);
convertFrom(bb, operands[0]);
datasetStack.pop();
} else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) {
SqlSampleSpec.SqlTableSampleSpec tableSampleSpec =
- (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
+ (SqlSampleSpec.SqlTableSampleSpec) sampleSpec;
convertFrom(bb, operands[0]);
RelOptSamplingParameters params =
- new RelOptSamplingParameters(
- tableSampleSpec.isBernoulli(),
- tableSampleSpec.getSamplePercentage(),
- tableSampleSpec.isRepeatable(),
- tableSampleSpec.getRepeatableSeed());
+ new RelOptSamplingParameters(
+ tableSampleSpec.isBernoulli(),
+ tableSampleSpec.getSamplePercentage(),
+ tableSampleSpec.isRepeatable(),
+ tableSampleSpec.getRepeatableSeed());
bb.setRoot(new Sample(cluster, bb.root, params), false);
} else {
- throw Util.newInternal(
- "unknown TABLESAMPLE type: " + sampleSpec);
+ throw new AssertionError("unknown TABLESAMPLE type: " + sampleSpec);
}
return;
case IDENTIFIER:
- final SqlValidatorNamespace fromNamespace =
- validator.getNamespace(from).resolve();
- if (fromNamespace.getNode() != null) {
- convertFrom(bb, fromNamespace.getNode());
- return;
- }
- final String datasetName =
- datasetStack.isEmpty() ? null : datasetStack.peek();
- boolean[] usedDataset = {false};
- RelOptTable table =
- SqlValidatorUtil.getRelOptTable(
- fromNamespace,
- catalogReader,
- datasetName,
- usedDataset);
- final RelNode tableRel;
- if (config.isConvertTableAccess()) {
- tableRel = toRel(table);
- } else {
- tableRel = LogicalTableScan.create(cluster, table);
- }
- bb.setRoot(tableRel, true);
- if (usedDataset[0]) {
- bb.setDataset(datasetName);
- }
+ convertIdentifier(bb, (SqlIdentifier) from, null);
+ return;
+
+ case EXTEND:
+ call = (SqlCall) from;
+ SqlIdentifier id = (SqlIdentifier) call.getOperandList().get(0);
+ SqlNodeList extendedColumns = (SqlNodeList) call.getOperandList().get(1);
+ convertIdentifier(bb, id, extendedColumns);
return;
case JOIN:
@@ -2056,15 +2044,15 @@ public class SqlToRelConverter {
final boolean isNatural = join.isNatural();
final JoinType joinType = join.getJoinType();
final SqlValidatorScope leftScope =
- Util.first(validator.getJoinScope(left),
- ((DelegatingScope) bb.scope).getParent());
+ Util.first(validator.getJoinScope(left),
+ ((DelegatingScope) bb.scope).getParent());
final Blackboard leftBlackboard =
- createBlackboard(leftScope, null, false);
+ createBlackboard(leftScope, null, false);
final SqlValidatorScope rightScope =
- Util.first(validator.getJoinScope(right),
- ((DelegatingScope) bb.scope).getParent());
+ Util.first(validator.getJoinScope(right),
+ ((DelegatingScope) bb.scope).getParent());
final Blackboard rightBlackboard =
- createBlackboard(rightScope, null, false);
+ createBlackboard(rightScope, null, false);
convertFrom(leftBlackboard, left);
RelNode leftRel = leftBlackboard.root;
convertFrom(rightBlackboard, right);
@@ -2077,29 +2065,29 @@ public class SqlToRelConverter {
final RelDataType leftRowType = leftNamespace.getRowType();
final RelDataType rightRowType = rightNamespace.getRowType();
final List<String> columnList =
- SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType,
- rightRowType);
+ SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType,
+ rightRowType);
conditionExp = convertUsing(leftNamespace, rightNamespace,
- columnList);
+ columnList);
} else {
conditionExp =
- convertJoinCondition(
- fromBlackboard,
- leftNamespace,
- rightNamespace,
- join.getCondition(),
- join.getConditionType(),
- leftRel,
- rightRel);
+ convertJoinCondition(
+ fromBlackboard,
+ leftNamespace,
+ rightNamespace,
+ join.getCondition(),
+ join.getConditionType(),
+ leftRel,
+ rightRel);
}
final RelNode joinRel =
- createJoin(
- fromBlackboard,
- leftRel,
- rightRel,
- conditionExp,
- convertedJoinType);
+ createJoin(
+ fromBlackboard,
+ leftRel,
+ rightRel,
+ conditionExp,
+ convertedJoinType);
bb.setRoot(joinRel, false);
return;
@@ -2129,13 +2117,13 @@ public class SqlToRelConverter {
fieldNames.add(validator.deriveAlias(node.e, node.i));
}
final RelNode input =
- RelOptUtil.createProject(
- (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
- exprs, fieldNames, true);
+ RelOptUtil.createProject(
+ (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster),
+ exprs, fieldNames, true);
Uncollect uncollect =
- new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
- input, operator.withOrdinality);
+ new Uncollect(cluster, cluster.traitSetOf(Convention.NONE),
+ input, operator.withOrdinality);
bb.setRoot(uncollect, true);
return;
@@ -2149,13 +2137,116 @@ public class SqlToRelConverter {
return;
default:
- throw Util.newInternal("not a join operator " + from);
+ throw new AssertionError("not a join operator " + from);
+ }
+ }
+
+ protected void convertMatchRecognize(Blackboard bb, SqlCall call) {
+ final SqlMatchRecognize matchRecognize = (SqlMatchRecognize) call;
+ final SqlValidatorNamespace ns = validator.getNamespace(matchRecognize);
+ final SqlValidatorScope scope = validator.getMatchRecognizeScope(matchRecognize);
+
+ final Blackboard mrBlackBoard = createBlackboard(scope, null, false);
+ final RelDataType rowType = ns.getRowType();
+ // convert inner query, could be a table name or a derived table
+ SqlNode expr = matchRecognize.getTableRef();
+ convertFrom(mrBlackBoard, expr);
+ final RelNode input = mrBlackBoard.root;
+
+ // convert pattern
+ final Set<String> patternVarsSet = new HashSet<>();
+ SqlNode pattern = matchRecognize.getPattern();
+ final SqlBasicVisitor<RexNode> patternVarVisitor =
+ new SqlBasicVisitor<RexNode>() {
+ @Override public RexNode visit(SqlCall call) {
+ List<SqlNode> operands = call.getOperandList();
+ List<RexNode> newOperands = Lists.newArrayList();
+ for (SqlNode node : operands) {
+ newOperands.add(node.accept(this));
+ }
+ return rexBuilder.makeCall(
+ validator.getUnknownType(), call.getOperator(), newOperands);
+ }
+
+ @Override public RexNode visit(SqlIdentifier id) {
+ assert id.isSimple();
+ patternVarsSet.add(id.getSimple());
+ return rexBuilder.makeLiteral(id.getSimple());
+ }
+
+ @Override public RexNode visit(SqlLiteral literal) {
+ if (literal instanceof SqlNumericLiteral) {
+ return rexBuilder.makeExactLiteral(BigDecimal.valueOf(literal.intValue(true)));
+ } else {
+ return rexBuilder.makeLiteral(literal.booleanValue());
+ }
+ }
+ };
+ final RexNode patternNode = pattern.accept(patternVarVisitor);
+
+ mrBlackBoard.setPatternVarRef(true);
+
+ // convert definitions
+ final ImmutableMap.Builder<String, RexNode> definitionNodes =
+ ImmutableMap.builder();
+ for (SqlNode def : matchRecognize.getPatternDefList()) {
+ List<SqlNode> operands = ((SqlCall) def).getOperandList();
+ String alias = ((SqlIdentifier) operands.get(1)).getSimple();
+ RexNode rex = mrBlackBoard.convertExpression(operands.get(0));
+ definitionNodes.put(alias, rex);
+ }
+
+ mrBlackBoard.setPatternVarRef(false);
+
+ final RelFactories.MatchFactory factory =
+ RelFactories.DEFAULT_MATCH_FACTORY;
+ final RelNode rel =
+ factory.createMatchRecognize(input, patternNode,
+ matchRecognize.getStrictStart().booleanValue(),
+ matchRecognize.getStrictEnd().booleanValue(),
+ definitionNodes.build(),
+ rowType);
+ bb.setRoot(rel, false);
+ }
+
+ private void convertIdentifier(Blackboard bb, SqlIdentifier id,
+ SqlNodeList extendedColumns) {
+ final SqlValidatorNamespace fromNamespace =
+ validator.getNamespace(id).resolve();
+ if (fromNamespace.getNode() != null) {
+ convertFrom(bb, fromNamespace.getNode());
+ return;
+ }
+ final String datasetName =
+ datasetStack.isEmpty() ? null : datasetStack.peek();
+ final boolean[] usedDataset = {false};
+ RelOptTable table =
+ SqlValidatorUtil.getRelOptTable(fromNamespace, catalogReader,
+ datasetName, usedDataset);
+ if (extendedColumns != null && extendedColumns.size() > 0) {
+ assert table != null;
+ final SqlValidatorTable validatorTable =
+ table.unwrap(SqlValidatorTable.class);
+ final List<RelDataTypeField> extendedFields =
+ SqlValidatorUtil.getExtendedColumns(validator, validatorTable,
+ extendedColumns);
+ table = table.extend(extendedFields);
+ }
+ final RelNode tableRel;
+ if (config.isConvertTableAccess()) {
+ tableRel = toRel(table);
+ } else {
+ tableRel = LogicalTableScan.create(cluster, table);
+ }
+ bb.setRoot(tableRel, true);
+ if (usedDataset[0]) {
+ bb.setDataset(datasetName);
}
}
protected void convertCollectionTable(
- Blackboard bb,
- SqlCall call) {
+ Blackboard bb,
+ SqlCall call) {
final SqlOperator operator = call.getOperator();
if (operator == SqlStdOperatorTable.TABLESAMPLE) {
final String sampleName = (String) SqlLiteral.value(call.operand(0));
@@ -2172,15 +2263,15 @@ public class SqlToRelConverter {
// Expand table macro if possible. It's more efficient than
// LogicalTableFunctionScan.
final SqlCallBinding callBinding =
- new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
+ new SqlCallBinding(bb.scope.getValidator(), bb.scope, call);
if (operator instanceof SqlUserDefinedTableMacro) {
final SqlUserDefinedTableMacro udf =
- (SqlUserDefinedTableMacro) operator;
+ (SqlUserDefinedTableMacro) operator;
final TranslatableTable table =
- udf.getTable(typeFactory, callBinding.operands());
+ udf.getTable(typeFactory, callBinding.operands());
final RelDataType rowType = table.getRowType(typeFactory);
RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table,
- udf.getNameAsId().names);
+ udf.getNameAsId().names);
RelNode converted = toRel(relOptTable);
bb.setRoot(converted, true);
return;
@@ -2197,23 +2288,23 @@ public class SqlToRelConverter {
RexNode rexCall = bb.convertExpression(call);
final List<RelNode> inputs = bb.retrieveCursors();
Set<RelColumnMapping> columnMappings =
- getColumnMappings(operator);
+ getColumnMappings(operator);
LogicalTableFunctionScan callRel =
- LogicalTableFunctionScan.create(
- cluster,
- inputs,
- rexCall,
- elementType,
- validator.getValidatedNodeType(call),
- columnMappings);
+ LogicalTableFunctionScan.create(
+ cluster,
+ inputs,
+ rexCall,
+ elementType,
+ validator.getValidatedNodeType(call),
+ columnMappings);
bb.setRoot(callRel, true);
afterTableFunction(bb, call, callRel);
}
protected void afterTableFunction(
- SqlToRelConverter.Blackboard bb,
- SqlCall call,
- LogicalTableFunctionScan callRel) {
+ SqlToRelConverter.Blackboard bb,
+ SqlCall call,
+ LogicalTableFunctionScan callRel) {
}
private Set<RelColumnMapping> getColumnMappings(SqlOperator op) {
@@ -2223,7 +2314,7 @@ public class SqlToRelConverter {
}
if (rti instanceof TableFunctionReturnTypeInference) {
TableFunctionReturnTypeInference tfrti =
- (TableFunctionReturnTypeInference) rti;
+ (TableFunctionReturnTypeInference) rti;
return tfrti.getColumnMappings();
} else {
return null;
@@ -2231,35 +2322,35 @@ public class SqlToRelConverter {
}
protected RelNode createJoin(
- Blackboard bb,
- RelNode leftRel,
- RelNode rightRel,
- RexNode joinCond,
- JoinRelType joinType) {
+ Blackboard bb,
+ RelNode leftRel,
+ RelNode rightRel,
+ RexNode joinCond,
+ JoinRelType joinType) {
assert joinCond != null;
final CorrelationUse p = getCorrelationUse(bb, rightRel);
if (p != null) {
LogicalCorrelate corr = LogicalCorrelate.create(leftRel, p.r,
- p.id, p.requiredColumns, SemiJoinType.of(joinType));
+ p.id, p.requiredColumns, SemiJoinType.of(joinType));
if (!joinCond.isAlwaysTrue()) {
final RelFactories.FilterFactory factory =
- RelFactories.DEFAULT_FILTER_FACTORY;
+ RelFactories.DEFAULT_FILTER_FACTORY;
return factory.createFilter(corr, joinCond);
}
return corr;
}
final Join originalJoin =
- (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
- joinCond, ImmutableSet.<CorrelationId>of(), joinType, false);
+ (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel,
+ joinCond, ImmutableSet.<CorrelationId>of(), joinType, false);
return RelOptUtil.pushDownJoinConditions(originalJoin);
}
private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) {
final Set<CorrelationId> correlatedVariables =
- RelOptUtil.getVariablesUsed(r0);
+ RelOptUtil.getVariablesUsed(r0);
if (correlatedVariables.isEmpty()) {
return null;
}
@@ -2274,18 +2365,21 @@ public class SqlToRelConverter {
for (CorrelationId correlName : correlatedVariables) {
DeferredLookup lookup =
- mapCorrelToDeferred.get(correlName);
+ mapCorrelToDeferred.get(correlName);
RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName);
String originalRelName = lookup.getOriginalRelName();
String originalFieldName = fieldAccess.getField().getName();
- SqlValidatorScope.ResolvedImpl resolved =
- new SqlValidatorScope.ResolvedImpl();
- lookup.bb.scope.resolve(ImmutableList.of(originalRelName), false,
- resolved);
+ final SqlNameMatcher nameMatcher =
+ lookup.bb.scope.getValidator().getCatalogReader().nameMatcher();
+ final SqlValidatorScope.ResolvedImpl resolved =
+ new SqlValidatorScope.ResolvedImpl();
+ lookup.bb.scope.resolve(ImmutableList.of(originalRelName),
+ nameMatcher, false, resolved);
assert resolved.count() == 1;
final SqlValidatorScope.Resolve resolve = resolved.only();
final SqlValidatorNamespace foundNs = resolve.namespace;
+
<TRUNCATED>