You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/03/11 17:17:22 UTC
[05/14] kylin git commit: KYLIN-2495 query exception when integer
column encoded as date/time encoding
KYLIN-2495 query exception when integer column encoded as date/time encoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5f3b228
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5f3b228
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5f3b228
Branch: refs/heads/KYLIN-2360
Commit: c5f3b2286987d12c31310e9c9670d469606e9d4c
Parents: 70dda40
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 15:57:10 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 17:17:02 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/DateFormat.java | 16 ++---
.../apache/kylin/cube/model/RowKeyColDesc.java | 7 +--
.../kylin/dimension/AbstractDateDimEnc.java | 25 ++++----
.../org/apache/kylin/dimension/DateDimEnc.java | 64 ++++++++++++++++----
.../org/apache/kylin/dimension/TimeDimEnc.java | 13 ++--
.../kylin/measure/topn/TopNMeasureType.java | 12 +++-
.../apache/kylin/dimension/DateDimEncTest.java | 2 +-
7 files changed, 88 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 2412543..29858f1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -48,6 +48,10 @@ public class DateFormat {
}
return r;
}
+
+ public static String formatToCompactDateStr(long millis) {
+ return formatToDateStr(millis, COMPACT_DATE_PATTERN);
+ }
public static String formatToDateStr(long millis) {
return formatToDateStr(millis, DEFAULT_DATE_PATTERN);
@@ -88,18 +92,6 @@ public class DateFormat {
}
public static long stringToMillis(String str) {
- return stringToMillis(str, null);
- }
-
- public static long stringToMillis(String str, String dateFormat) {
- try {
- if (dateFormat != null) {
- return getDateFormat(dateFormat).parse(str).getTime();
- }
- } catch (ParseException e) {
- // given format does not work, proceed to below
- }
-
// try to be smart and guess the date format
if (isAllDigits(str)) {
if (str.length() == 8)
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 1df73ad..b6f0a27 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -86,10 +86,9 @@ public class RowKeyColDesc implements java.io.Serializable {
encoding = encodingName = TimeDimEnc.ENCODING_NAME;
}
}
- // if (DateDimEnc.ENCODING_NAME.equals(encodingName) && type.isDate() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply date encoding");
- // if (TimeDimEnc.ENCODING_NAME.equals(encodingName) && type.isTimeFamily() == false)
- // throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply time encoding");
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, type);
+
if (encodingName.startsWith(FixedLenDimEnc.ENCODING_NAME) && (type.isIntegerFamily() || type.isNumberFamily()))
throw new IllegalArgumentException(colRef + " type is " + type + " and cannot apply fixed_length encoding");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
index a54bcda..9c0d0da 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java
@@ -18,10 +18,6 @@
package org.apache.kylin.dimension;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -29,20 +25,23 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+
public class AbstractDateDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
- interface IMillisCodec extends Serializable {
- long millisToCode(long millis);
+ interface IValueCodec extends Serializable {
+ long valueToCode(String value);
- long codeToMillis(long code);
+ String codeToValue(long code);
}
// ============================================================================
private int fixedLen;
- private IMillisCodec codec;
+ private IValueCodec codec;
- protected AbstractDateDimEnc(int fixedLen, IMillisCodec codec) {
+ protected AbstractDateDimEnc(int fixedLen, IValueCodec codec) {
this.fixedLen = fixedLen;
this.codec = codec;
}
@@ -59,8 +58,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
return;
}
- long millis = DateFormat.stringToMillis(value);
- long code = codec.millisToCode(millis);
+ long code = codec.valueToCode(value);
BytesUtil.writeLong(code, output, outputOffset, fixedLen);
}
@@ -74,8 +72,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
if (code < 0)
throw new IllegalArgumentException();
- long millis = codec.codeToMillis(code);
- return String.valueOf(millis);
+ return codec.codeToValue(code);
}
@Override
@@ -137,7 +134,7 @@ public class AbstractDateDimEnc extends DimensionEncoding {
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
this.fixedLen = in.readInt();
- this.codec = (IMillisCodec) in.readObject();
+ this.codec = (IValueCodec) in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 3b390d9..fee6233 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -20,10 +20,13 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
+
/**
* This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility.
*/
-public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
+public class DateDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final int ID_9999_12_31 = 3652426;
@@ -38,24 +41,47 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
@Override
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
- return new DateDimEnc();
+ return new DateDimEnc(args);
}
};
- public DateDimEnc() {
- super(3, new IMillisCodec() {
- private static final long serialVersionUID = 1L;
+ private static class DateDimValueCodec implements IValueCodec {
+
+ private static final long serialVersionUID = 1L;
+ private DataType datatype = null;
- @Override
- public long millisToCode(long millis) {
- return getNumOfDaysSince0000FromMillis(millis);
+ public DateDimValueCodec(String[] args) {
+ if (args != null && args.length == 1) {
+ datatype = DataType.getType(args[0]);
}
+ }
+
+ @Override
+ public long valueToCode(String value) {
+ //if data type is integer, DateFormat.stringToMillis recognizes format like "20001010"
+ long millis = DateFormat.stringToMillis(value);
+
+ return getNumOfDaysSince0000FromMillis(millis);
+ }
- @Override
- public long codeToMillis(long code) {
- return getMillisFromNumOfDaysSince0000(code);
+ @Override
+ public String codeToValue(long code) {
+ long millisFromNumOfDaysSince0000 = getMillisFromNumOfDaysSince0000(code);
+ if (datatype != null && datatype.isIntegerFamily()) {
+ return DateFormat.formatToCompactDateStr(millisFromNumOfDaysSince0000);
+ } else {
+ return String.valueOf(millisFromNumOfDaysSince0000);
}
- });
+ }
+ }
+
+ //keep this for ser/der
+ public DateDimEnc() {
+ super(3, new DateDimValueCodec(null));
+ }
+
+ public DateDimEnc(String[] args) {
+ super(3, new DateDimValueCodec(args));
}
public static long getNumOfDaysSince0000FromMillis(long millis) {
@@ -69,4 +95,18 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
return millis;
}
+ public static String[] replaceEncodingArgs(String encoding, String[] encodingArgs, String encodingName, DataType type) {
+ // https://issues.apache.org/jira/browse/KYLIN-2495
+ if (DateDimEnc.ENCODING_NAME.equals(encodingName)) {
+ if (type.isIntegerFamily()) {
+ if (encodingArgs.length != 0) {
+ throw new IllegalArgumentException("Date encoding should not specify arguments: " + encoding);
+ }
+ return new String[] { type.toString() };
+ }
+ }
+
+ return encodingArgs;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index 4bef2b8..dcc99dd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -20,10 +20,12 @@ package org.apache.kylin.dimension;
import java.io.Serializable;
+import org.apache.kylin.common.util.DateFormat;
+
/**
* This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility.
*/
-public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
+public class TimeDimEnc extends AbstractDateDimEnc implements Serializable {
private static final long serialVersionUID = 1L;
public static final String ENCODING_NAME = "time";
@@ -41,17 +43,18 @@ public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
};
public TimeDimEnc() {
- super(4, new IMillisCodec() {
+ super(4, new IValueCodec() {
private static final long serialVersionUID = 1L;
@Override
- public long millisToCode(long millis) {
+ public long valueToCode(String value) {
+ long millis = DateFormat.stringToMillis(value);
return millis / 1000;
}
@Override
- public long codeToMillis(long code) {
- return code * 1000;
+ public String codeToValue(long code) {
+ return String.valueOf(code * 1000);
}
});
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 8c8b5a6..b7252a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -24,10 +24,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DateDimEnc;
import org.apache.kylin.dimension.DictionaryDimEnc;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -47,6 +47,8 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
private static final Logger logger = LoggerFactory.getLogger(TopNMeasureType.class);
@@ -155,7 +157,6 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
return topNCounter;
}
-
@Override
public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
TopNCounter<ByteArray> topNCounter = value;
@@ -424,7 +425,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
}
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
- dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion);
+ String encodingName = (String) encodingConf[0];
+ String[] encodingArgs = (String[]) encodingConf[1];
+
+ encodingArgs = DateDimEnc.replaceEncodingArgs(encoding, encodingArgs, encodingName, literalCols.get(i).getType());
+
+ dimensionEncodings[i] = DimensionEncodingFactory.create(encodingName, encodingArgs, encodingVersion);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5f3b228/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
index 0183b5f..082f3ce 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/DateDimEncTest.java
@@ -38,7 +38,7 @@ public class DateDimEncTest {
@Before
public void setup() {
- enc = new DateDimEnc();
+ enc = new DateDimEnc(null);
buf = new byte[enc.getLengthOfEncoding()];
}