You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/11/21 07:45:04 UTC

[1/3] kylin git commit: minor refactors to IGTCodeSystem

Repository: kylin
Updated Branches:
  refs/heads/master a179f5e83 -> d38d2e955


minor refactors to IGTCodeSystem


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b525d07
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b525d07
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b525d07

Branch: refs/heads/master
Commit: 4b525d074d82bccb89b0dad0cfa75f198ad3c4f0
Parents: 3ab966b
Author: Hongbin Ma <ma...@apache.org>
Authored: Sat Nov 19 22:10:51 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:58 2016 +0800

----------------------------------------------------------------------
 .../cube/gridtable/TrimmedCubeCodeSystem.java   |  8 ++++---
 .../java/org/apache/kylin/gridtable/GTInfo.java | 24 ++++++++++++--------
 .../kylin/gridtable/GTSampleCodeSystem.java     |  7 +++---
 3 files changed, 24 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index 86c44e9..a0b230e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -30,6 +30,7 @@ import org.apache.kylin.common.util.BytesSerializer;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.dimension.DictionaryDimEnc;
 import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.gridtable.IGTCodeSystem;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
 import com.google.common.collect.Maps;
@@ -87,9 +88,10 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
         }
     }
 
-    public static final BytesSerializer<TrimmedCubeCodeSystem> serializer = new BytesSerializer<TrimmedCubeCodeSystem>() {
+    public static final BytesSerializer<IGTCodeSystem> serializer = new BytesSerializer<IGTCodeSystem>() {
         @Override
-        public void serialize(TrimmedCubeCodeSystem value, ByteBuffer out) {
+        public void serialize(IGTCodeSystem ivalue, ByteBuffer out) {
+            TrimmedCubeCodeSystem value = (TrimmedCubeCodeSystem) ivalue;
             BytesUtil.writeVInt(value.dependentMetricsMap.size(), out);
             for (Map.Entry<Integer, Integer> x : value.dependentMetricsMap.entrySet()) {
                 BytesUtil.writeVInt(x.getKey(), out);
@@ -105,7 +107,7 @@ public class TrimmedCubeCodeSystem extends CubeCodeSystem {
         }
 
         @Override
-        public TrimmedCubeCodeSystem deserialize(ByteBuffer in) {
+        public IGTCodeSystem deserialize(ByteBuffer in) {
             Map<Integer, Integer> dependentMetricsMap = Maps.newHashMap();
 
             int size = BytesUtil.readVInt(in);

http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 12a3593..ba62af3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -290,11 +290,17 @@ public class GTInfo {
                 BytesUtil.writeAsciiString(CubeCodeSystem.class.getCanonicalName(), out);
                 TrimmedCubeCodeSystem trimmed = ((CubeCodeSystem) value.codeSystem).trimForCoprocessor();
                 TrimmedCubeCodeSystem.serializer.serialize(trimmed, out);
-            } else if (value.codeSystem instanceof GTSampleCodeSystem) {
-                BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out);
-                GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) value.codeSystem, out);
-            } else {
+            } else if (value.codeSystem != null) {
                 BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out);
+                BytesSerializer<IGTCodeSystem> serializer = null;
+                try {
+                    serializer = (BytesSerializer<IGTCodeSystem>) value.codeSystem.getClass().getField("serializer").get(null);
+                } catch (IllegalAccessException | NoSuchFieldException e) {
+                    throw new RuntimeException("failed to get serializer for " + value.codeSystem.getClass(), e);
+                }
+                serializer.serialize(value.codeSystem, out);
+            } else {
+                throw new IllegalStateException("code system cannot be null");
             }
 
             BytesUtil.writeUTFString(value.tableName, out);
@@ -317,13 +323,13 @@ public class GTInfo {
             String codeSystemType = BytesUtil.readAsciiString(in);
             if (CubeCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
                 codeSystem = TrimmedCubeCodeSystem.serializer.deserialize(in);
-            } else if (GTSampleCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
-                codeSystem = GTSampleCodeSystem.serializer.deserialize(in);
             } else {
                 try {
-                    codeSystem = (IGTCodeSystem) Class.forName(codeSystemType).newInstance();
-                } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
-                    throw new RuntimeException(e);
+                    Class clazz = Class.forName(codeSystemType);
+                    BytesSerializer<IGTCodeSystem> serializer = (BytesSerializer<IGTCodeSystem>) clazz.getField("serializer").get(null);
+                    codeSystem = serializer.deserialize(in);
+                } catch (Exception e) {
+                    throw new RuntimeException("Failed to deserialize IGTCodeSystem " + codeSystemType, e);
                 }
             }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/4b525d07/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
index e379c42..3f3c844 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTSampleCodeSystem.java
@@ -106,13 +106,14 @@ public class GTSampleCodeSystem implements IGTCodeSystem {
         return serializers[col].deserialize(buf);
     }
 
-    public static final BytesSerializer<GTSampleCodeSystem> serializer = new BytesSerializer<GTSampleCodeSystem>() {
+    @SuppressWarnings("unused") //used by reflection
+    public static final BytesSerializer<IGTCodeSystem> serializer = new BytesSerializer<IGTCodeSystem>() {
         @Override
-        public void serialize(GTSampleCodeSystem value, ByteBuffer out) {
+        public void serialize(IGTCodeSystem value, ByteBuffer out) {
         }
 
         @Override
-        public GTSampleCodeSystem deserialize(ByteBuffer in) {
+        public IGTCodeSystem deserialize(ByteBuffer in) {
             return new GTSampleCodeSystem();
         }
     };


[2/3] kylin git commit: KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep

Posted by ma...@apache.org.
KYLIN-2210 call CubeStatsReader.print at SaveStatisticsStep


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ab966b6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ab966b6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ab966b6

Branch: refs/heads/master
Commit: 3ab966b650debb83eb219c9ed8d357d141466776
Parents: a179f5e
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Nov 18 14:56:30 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:58 2016 +0800

----------------------------------------------------------------------
 .../kylin/engine/mr/common/CubeStatsReader.java    |  2 +-
 .../kylin/engine/mr/steps/SaveStatisticsStep.java  | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..c6839d6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -227,7 +227,7 @@ public class CubeStatsReader {
         return ret;
     }
 
-    private void print(PrintWriter out) {
+    public void print(PrintWriter out) {
         Map<Long, Long> cuboidRows = getCuboidRowEstimatesHLL();
         Map<Long, Double> cuboidSizes = getCuboidSizeMap();
         List<Long> cuboids = new ArrayList<Long>(cuboidRows.keySet());

http://git-wip-us.apache.org/repos/asf/kylin/blob/3ab966b6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 23e81bc..7718bfb 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,6 +19,8 @@
 package org.apache.kylin.engine.mr.steps;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.Random;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -29,8 +31,8 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.ResourceStore;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
+import org.apache.kylin.engine.mr.HadoopUtil;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.engine.mr.common.CubeStatsReader;
 import org.apache.kylin.job.exception.ExecuteException;
@@ -85,6 +87,16 @@ public class SaveStatisticsStep extends AbstractExecutable {
 
     private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException {
         String algPref = kylinConf.getCubeAlgorithm();
+
+        CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
+        StringWriter sw = new StringWriter();
+        PrintWriter pw = new PrintWriter(sw);
+        cubeStats.print(pw);
+        pw.flush();
+        pw.close();
+        logger.info("Cube Stats Estimation for segment {} :", seg.toString());
+        logger.info(sw.toString());
+
         AlgorithmEnum alg;
         if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
             alg = AlgorithmEnum.INMEM;
@@ -104,14 +116,13 @@ public class SaveStatisticsStep extends AbstractExecutable {
             } else if ("random".equalsIgnoreCase(algPref)) { // for testing
                 alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
             } else { // the default
-                CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
                 int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
                 int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
                 double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
                 double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
                 logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
                 logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
- 
+
                 // in-mem cubing is good when
                 // 1) the cluster has enough mapper slots to run in parallel
                 // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage


[3/3] kylin git commit: KYLIN-1875 minor bug fix

Posted by ma...@apache.org.
KYLIN-1875 minor bug fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d38d2e95
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d38d2e95
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d38d2e95

Branch: refs/heads/master
Commit: d38d2e9550a0a296ab7edbbbbb0155cfa39fc21f
Parents: 4b525d0
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Nov 21 13:37:20 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Nov 21 13:37:59 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/util/DateFormat.java    | 10 ++++++--
 .../org/apache/kylin/metadata/tuple/Tuple.java  | 27 ++++++++++----------
 .../apache/kylin/query/schema/OLAPTable.java    |  2 +-
 3 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 1d70a2d..2412543 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -103,6 +103,7 @@ public class DateFormat {
         // try to be smart and guess the date format
         if (isAllDigits(str)) {
             if (str.length() == 8)
+                //TODO: might be prolematic if an actual ts happends to be 8 digits, e.g. 1970-01-01 10:00:01.123
                 return stringToDate(str, COMPACT_DATE_PATTERN).getTime();
             else
                 return Long.parseLong(str);
@@ -119,8 +120,13 @@ public class DateFormat {
 
     private static boolean isAllDigits(String str) {
         for (int i = 0, n = str.length(); i < n; i++) {
-            if (Character.isDigit(str.charAt(i)) == false)
-                return false;
+            if (!Character.isDigit(str.charAt(i))) {
+                if (i == 0 && str.charAt(0) == '-') {
+                    continue;
+                } else {
+                    return false;
+                }
+            }
         }
         return true;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 9074403..721a719 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -181,29 +181,30 @@ public class Tuple implements ITuple {
         if ((strValue.equals("") || strValue.equals("\\N")) && !dataTypeName.equals("string") && !dataTypeName.startsWith("varchar"))
             return null;
 
-        // TODO use data type enum instead of string comparison
-        if ("date".equals(dataTypeName)) {
+        switch (dataTypeName) {
+        case "date":
             // convert epoch time
             return Integer.valueOf(dateToEpicDays(strValue));// Optiq expects Integer instead of Long. by honma
-        } else if ("timestamp".equals(dataTypeName) || "datetime".equals(dataTypeName)) {
+        case "datetime":
+        case "timestamp":
             return Long.valueOf(DateFormat.stringToMillis(strValue));
-        } else if ("tinyint".equals(dataTypeName)) {
+        case "tinyint":
             return Byte.valueOf(strValue);
-        } else if ("smallint".equals(dataTypeName)) {
+        case "smallint":
             return Short.valueOf(strValue);
-        } else if ("integer".equals(dataTypeName)) {
+        case "integer":
             return Integer.valueOf(strValue);
-        } else if ("bigint".equals(dataTypeName)) {
+        case "bigint":
             return Long.valueOf(strValue);
-        } else if ("double".equals(dataTypeName)) {
+        case "double":
             return Double.valueOf(strValue);
-        } else if ("decimal".equals(dataTypeName)) {
+        case "decimal":
             return new BigDecimal(strValue);
-        } else if ("float".equals(dataTypeName)) {
+        case "float":
             return Float.valueOf(strValue);
-        } else if ("boolean".equals(dataTypeName)) {
-            return Boolean.valueOf(strValue);
-        } else {
+        case "boolean":
+            return Boolean.valueOf(strValue) | "1".equals(strValue); // in some extended encodings boolean might be encoded as a number
+        default:
             return strValue;
         }
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/d38d2e95/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 21b60f1..06ffdba 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -203,7 +203,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
             if (realization.getType() == RealizationType.INVERTED_INDEX && realization.getModel().isFactTable(sourceTable.getIdentity())) {
                 DataModelDesc dataModelDesc = realization.getModel();
                 for (String metricColumn : dataModelDesc.getMetrics()) {
-                    ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getRootFactTable() + "." + metricColumn);
+                    ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getRootFactTable().getTableIdentity() + "." + metricColumn);
                     if (columnDesc.getType().isIntegerFamily() && !columnDesc.getType().isBigInt())
                         updateColumns.add(columnDesc);
                 }