You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/11/21 07:45:04 UTC
[1/3] kylin git commit: minor refactors to IGTCodeSystem
Repository: kylin
Updated Branches:
refs/heads/master a179f5e83 -> d38d2e955
minor refactors to IGTCodeSystem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b525d07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b525d07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b525d07
Branch: refs/heads/master
Commit: 4b525d074d82bccb89b0dad0cfa75f198ad3c4f0
Parents: 3ab966b
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Nov 19 22:10:51 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:58 2016 +0800
----------------------------------------------------------------------
.../cube/gridtable/TrimmedCubeCodeSystem.java | 8 ++++---
.../java/org/apache/kylin/gridtable/GTInfo.java | 24 ++++++++++++--------
.../kylin/gridtable/GTSampleCodeSystem.java | 7 +++---
3 files changed, 24 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index 86c44e9..a0b230e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesSerializer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.IGTCodeSystem;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import com.google.common.collect.Maps;
@@ -87,9 +88,10 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
}
}
- public static final BytesSerializer<TrimmedCubeCodeSystem> serializer = new BytesSerializer<TrimmedCubeCodeSystem>() {
+ public static final BytesSerializer<IGTCodeSystem> serializer = new BytesSerializer<IGTCodeSystem>() {
@Override
- public void serialize(TrimmedCubeCodeSystem value, ByteBuffer out) {
+ public void serialize(IGTCodeSystem ivalue, ByteBuffer out) {
+ TrimmedCubeCodeSystem value = (TrimmedCubeCodeSystem) ivalue;
BytesUtil.writeVInt(value.dependentMetricsMap.size(), out);
for (Map.Entry<Integer, Integer> x : value.dependentMetricsMap.entrySet()) {
BytesUtil.writeVInt(x.getKey(), out);
@@ -105,7 +107,7 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
}
@Override
- public TrimmedCubeCodeSystem deserialize(ByteBuffer in) {
+ public IGTCodeSystem deserialize(ByteBuffer in) {
Map<Integer, Integer> dependentMetricsMap = Maps.newHashMap();
int size = BytesUtil.readVInt(in);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 12a3593..ba62af3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -290,11 +290,17 @@ public class GTInfo {
BytesUtil.writeAsciiString(CubeCodeSystem.class.getCanonicalName(), out);
TrimmedCubeCodeSystem trimmed = ((CubeCodeSystem) value.codeSystem).trimForCoprocessor();
TrimmedCubeCodeSystem.serializer.serialize(trimmed, out);
- } else if (value.codeSystem instanceof GTSampleCodeSystem) {
- BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out);
- GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) value.codeSystem, out);
- } else {
+ } else if (value.codeSystem != null) {
BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out);
+ BytesSerializer<IGTCodeSystem> serializer = null;
+ try {
+ serializer = (BytesSerializer<IGTCodeSystem>) value.codeSystem.getClass().getField("serializer").get(null);
+ } catch (IllegalAccessException | NoSuchFieldException e) {
+ throw new RuntimeException("failed to get serializer for " + value.codeSystem.getClass(), e);
+ }
+ serializer.serialize(value.codeSystem, out);
+ } else {
+ throw new IllegalStateException("code system cannot be null");
}
BytesUtil.writeUTFString(value.tableName, out);
@@ -317,13 +323,13 @@ public class GTInfo {
String codeSystemType = BytesUtil.readAsciiString(in);
if (CubeCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
codeSystem = TrimmedCubeCodeSystem.serializer.deserialize(in);
- } else if (GTSampleCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
- codeSystem = GTSampleCodeSystem.serializer.deserialize(in);
} else {
try {
- codeSystem = (IGTCodeSystem) Class.forName(codeSystemType).newInstance();
- } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
- throw new RuntimeException(e);
+ Class clazz = Class.forName(codeSystemType);
+ BytesSerializer<IGTCodeSystem> serializer = (BytesSerializer<IGTCodeSystem>) clazz.getField("serializer").get(null);
+ codeSystem = serializer.deserialize(in);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to deserialize IGTCodeSystem " + codeSystemType, e);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index e379c42..3f3c844 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -106,13 +106,14 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
return serializers[col].deserialize(buf);
}
- public static final BytesSerializer<GTSampleCodeSystem> serializer = new BytesSerializer<GTSampleCodeSystem>() {
+ @SuppressWarnings("unused") //used by reflection
+ public static final BytesSerializer<IGTCodeSystem> serializer = new BytesSerializer<IGTCodeSystem>() {
@Override
- public void serialize(GTSampleCodeSystem value, ByteBuffer out) {
+ public void serialize(IGTCodeSystem value, ByteBuffer out) {
}
@Override
- public GTSampleCodeSystem deserialize(ByteBuffer in) {
+ public IGTCodeSystem deserialize(ByteBuffer in) {
return new GTSampleCodeSystem();
}
};
[2/3] kylin git commit: KYLIN-2210 call CubeStatsReader.print at
SaveStatisticsStep
Posted by ma...@apache.org.
KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ab966b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ab966b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ab966b6
Branch: refs/heads/master
Commit: 3ab966b650debb83eb219c9ed8d357d141466776
Parents: a179f5e
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Nov 18 14:56:30 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:58 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/CubeStatsReader.java | 2 +-
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 17 ++++++++++++++---
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..c6839d6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
return ret;
}
- private void print(PrintWriter out) {
+ public void print(PrintWriter out) {
Map<Long, Long> cuboidRows = getCuboidRowEstimatesHLL();
Map<Long, Double> cuboidSizes = getCuboidSizeMap();
List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet());
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 23e81bc..7718bfb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,6 +19,8 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.Random;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -29,8 +31,8 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
import org.apache.kylin.job.exception.ExecuteException;
@@ -85,6 +87,16 @@ public class SaveStatisticsStep extends AbstractExecutable {
private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException {
String algPref = kylinConf.getCubeAlgorithm();
+
+ CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ cubeStats.print(pw);
+ pw.flush();
+ pw.close();
+ logger.info("Cube Stats Estimation for segment {} :", seg.toString());
+ logger.info(sw.toString());
+
AlgorithmEnum alg;
if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
alg = AlgorithmEnum.INMEM;
@@ -104,14 +116,13 @@ public class SaveStatisticsStep extends AbstractExecutable {
} else if ("random".equalsIgnoreCase(algPref)) { // for testing
alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
} else { // the default
- CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
+
// in-mem cubing is good when
// 1) the cluster has enough mapper slots to run in parallel
// 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
[3/3] kylin git commit: KYLIN-1875 minor bug fix
Posted by ma...@apache.org.
KYLIN-1875 minor bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d38d2e95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d38d2e95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d38d2e95
Branch: refs/heads/master
Commit: d38d2e9550a0a296ab7edbbbbb0155cfa39fc21f
Parents: 4b525d0
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Nov 21 13:37:20 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:59 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 10 ++++++--
.../org/apache/kylin/metadata/tuple/Tuple.java | 27 ++++++++++----------
.../apache/kylin/query/schema/OLAPTable.java | 2 +-
3 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 1d70a2d..2412543 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -103,6 +103,7 @@ public class DateFormat {
// try to be smart and guess the date format
if (isAllDigits(str)) {
if (str.length() == 8)
+ //TODO: might be prolematic if an actual ts happends to be 8 digits, e.g. 1970-01-01 10:00:01.123
return stringToDate(str, COMPACT_DATE_PATTERN).getTime();
else
return Long.parseLong(str);
@@ -119,8 +120,13 @@ public class DateFormat {
private static boolean isAllDigits(String str) {
for (int i = 0, n = str.length(); i < n; i++) {
- if (Character.isDigit(str.charAt(i)) == false)
- return false;
+ if (!Character.isDigit(str.charAt(i))) {
+ if (i == 0 && str.charAt(0) == '-') {
+ continue;
+ } else {
+ return false;
+ }
+ }
}
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 9074403..721a719 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -181,29 +181,30 @@ public class Tuple implements ITuple {
if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
return null;
- // TODO use data type enum instead of string comparison
- if ("date".equals(dataTypeName)) {
+ switch (dataTypeName) {
+ case "date":
// convert epoch time
return Integer.valueOf(dateToEpicDays(strValue));// Optiq expects Integer instead of Long. by honma
- } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+ case "datetime":
+ case "timestamp":
return Long.valueOf(DateFormat.stringToMillis(strValue));
- } else if ("tinyint".equals(dataTypeName)) {
+ case "tinyint":
return Byte.valueOf(strValue);
- } else if ("smallint".equals(dataTypeName)) {
+ case "smallint":
return Short.valueOf(strValue);
- } else if ("integer".equals(dataTypeName)) {
+ case "integer":
return Integer.valueOf(strValue);
- } else if ("bigint".equals(dataTypeName)) {
+ case "bigint":
return Long.valueOf(strValue);
- } else if ("double".equals(dataTypeName)) {
+ case "double":
return Double.valueOf(strValue);
- } else if ("decimal".equals(dataTypeName)) {
+ case "decimal":
return new BigDecimal(strValue);
- } else if ("float".equals(dataTypeName)) {
+ case "float":
return Float.valueOf(strValue);
- } else if ("boolean".equals(dataTypeName)) {
- return Boolean.valueOf(strValue);
- } else {
+ case "boolean":
+ return Boolean.valueOf(strValue) | "1".equals(strValue); // in some extended encodings boolean might be encoded as a number
+ default:
return strValue;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 21b60f1..06ffdba 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -203,7 +203,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
if (realization.getType() == RealizationType.INVERTED_INDEX && realization.getModel().isFactTable(sourceTable.getIdentity())) {
DataModelDesc dataModelDesc = realization.getModel();
for (String metricColumn : dataModelDesc.getMetrics()) {
- ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getRootFactTable() + "." + metricColumn);
+ ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getRootFactTable().getTableIdentity() + "." + metricColumn);
if (columnDesc.getType().isIntegerFamily() && !columnDesc.getType().isBigInt())
updateColumns.add(columnDesc);
}