You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/15 05:06:30 UTC
[18/50] [abbrv] incubator-kylin git commit: KYLIN-749 pending no how
to deal with time
KYLIN-749 pending no how to deal with time
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/05e0bada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/05e0bada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/05e0bada
Branch: refs/heads/streaming-localdict
Commit: 05e0badad3a4263a64d37eece14507fd334af682
Parents: dcf0a21
Author: honma <ho...@ebay.com>
Authored: Wed May 6 12:35:44 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 7 15:04:48 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BasicTest.java | 19 +++++-----
.../apache/kylin/dict/DictionaryGenerator.java | 17 +++++----
.../apache/kylin/dict/NumberDictionaryTest.java | 20 +++++------
.../invertedindex/index/TableRecordInfo.java | 6 ++--
.../invertedindex/model/IIKeyValueCodec.java | 12 ++++---
.../kylin/job/dataGen/FactTableGenerator.java | 23 ++++++------
.../apache/kylin/metadata/model/DataType.java | 5 +++
.../kylin/query/relnode/OLAPFilterRel.java | 38 +++++---------------
.../apache/kylin/query/test/KylinQueryTest.java | 23 ++++++------
9 files changed, 76 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index 33f6141..db80934 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -18,19 +18,20 @@
package org.apache.kylin.common.util;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.*;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ranges;
import org.apache.commons.configuration.ConfigurationException;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+
/**
* Created by honma on 10/17/14.
* <p/>
@@ -62,6 +63,8 @@ public class BasicTest {
@Test
@Ignore("convenient trial tool for dev")
public void test1() throws Exception {
+ System.out.println(Long.parseLong("0032"));
+ System.out.println(time(1000L * Integer.MAX_VALUE));
System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(3, 10)));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index c4acf7f..59d7bb3 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -36,14 +36,14 @@ import java.util.*;
/**
* @author yangli9
*/
-@SuppressWarnings({"rawtypes", "unchecked"})
+@SuppressWarnings({ "rawtypes", "unchecked" })
public class DictionaryGenerator {
private static final int DICT_MAX_CARDINALITY = 2000000; // 2 million
private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
- private static final String[] DATE_PATTERNS = new String[]{"yyyy-MM-dd"};
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd" };
public static Dictionary<?> buildDictionaryFromValueList(DictionaryInfo info, Collection<byte[]> values) {
Dictionary<?> dict = buildDictionaryFromValueList(DataType.getInstance(info.getDataType()), values);
@@ -60,7 +60,7 @@ public class DictionaryGenerator {
// build dict, case by data type
if (dataType.isDateTimeFamily()) {
- dict = buildDateStrDict(values, baseId, nSamples, samples);
+ dict = buildDateTimeDict(values, baseId, nSamples, samples);
} else if (dataType.isNumberFamily()) {
dict = buildNumberDict(values, baseId, nSamples, samples);
} else {
@@ -77,7 +77,7 @@ public class DictionaryGenerator {
}
logger.debug("Dictionary value samples: " + buf.toString());
logger.debug("Dictionary cardinality " + dict.getSize());
- if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
+ if (dict instanceof TrieDictionary && dict.getSize() > DICT_MAX_CARDINALITY) {
throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- cardinality: " + values.size());
}
return dict;
@@ -116,7 +116,7 @@ public class DictionaryGenerator {
return buildDictionaryFromValueList(info, values);
}
- private static Dictionary buildDateStrDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
+ private static Dictionary buildDateTimeDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
final int BAD_THRESHOLD = 2;
String matchPattern = null;
@@ -142,10 +142,13 @@ public class DictionaryGenerator {
}
}
}
- if (matchPattern != null)
+ if (matchPattern != null) {
return new DateStrDictionary(matchPattern, baseId);
+ }
}
- throw new IllegalStateException("Unrecognized datetime value");
+
+ //FIXME: except for date type, all other date time family types are treated as string
+ return buildStringDict(values, baseId, nSamples, samples);
}
private static Dictionary buildStringDict(Collection<byte[]> values, int baseId, int nSamples, ArrayList samples) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index f9af244..405fece 100644
--- a/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -18,22 +18,17 @@
package org.apache.kylin.dict;
-import static org.junit.Assert.*;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.metadata.model.DataType;
import org.junit.Test;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import java.math.BigDecimal;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
/**
* @author yangli9
@@ -70,6 +65,7 @@ public class NumberDictionaryTest {
checkCodec("-12345.123", "-9999999999987654.876;");
checkCodec("0", "00000000000000000");
checkCodec("0.0", "00000000000000000.0");
+ checkCodec("123456789123456789", "-9999999999987654;");
}
private void checkCodec(String number, String code) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
index f2d847c..c57e91f 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
@@ -18,8 +18,6 @@
package org.apache.kylin.invertedindex.index;
-import java.util.List;
-
import org.apache.kylin.common.util.Array;
import org.apache.kylin.dict.Dictionary;
import org.apache.kylin.invertedindex.IISegment;
@@ -28,6 +26,8 @@ import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import java.util.List;
+
/**
* @author yangli9
* <p/>
@@ -80,7 +80,7 @@ public class TableRecordInfo {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 10;
+ lengths[i] = 23;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
index 7c1b802..4300140 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java
@@ -20,18 +20,22 @@ package org.apache.kylin.invertedindex.model;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.invertedindex.index.*;
+import org.apache.kylin.invertedindex.index.ColumnValueContainer;
+import org.apache.kylin.invertedindex.index.CompressedValueContainer;
+import org.apache.kylin.invertedindex.index.Slice;
+import org.apache.kylin.invertedindex.index.TableRecordInfoDigest;
import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
import org.apache.kylin.metadata.model.DataType;
import java.io.*;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
/**
* @author yangli9
@@ -139,7 +143,7 @@ public class IIKeyValueCodec implements KeyValueCodec {
} else if (dataType.isStringFamily()) {
lengths[i] = 256;
} else if (dataType.isDateTimeFamily()) {
- lengths[i] = 10;
+ lengths[i] = 23;
} else {
throw new RuntimeException("invalid data type:" + dataType);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 9bcaf1b..7cafea6 100644
--- a/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/job/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -18,13 +18,6 @@
package org.apache.kylin.job.dataGen;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Array;
@@ -32,12 +25,15 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataType;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.*;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
/**
* Created by hongbin on 5/20/14.
@@ -347,6 +343,9 @@ public class FactTableGenerator {
return String.format("%.4f", r.nextDouble() * (high - low) + low);
} else if (type.isDateTimeFamily()) {
+ if (!type.isDate()) {
+ throw new RuntimeException("Does not support " + type);
+ }
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
Date start = format.parse(range.get(0));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index f60db03..607b547 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -203,6 +203,11 @@ public class DataType {
return DATETIME_FAMILY.contains(name);
}
+ public boolean isDate()
+ {
+ return name.equals("date");
+ }
+
public boolean isTinyInt() {
return name.equals("tinyint");
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 16f2562..f15078a 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -18,50 +18,28 @@
package org.apache.kylin.query.relnode;
-import java.util.*;
-
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import net.hydromatic.optiq.rules.java.EnumerableConvention;
import net.hydromatic.optiq.rules.java.EnumerableRel;
import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
import net.hydromatic.optiq.rules.java.JavaRules.EnumerableCalcRel;
import net.hydromatic.optiq.runtime.SqlFunctions;
-
+import org.apache.kylin.metadata.filter.*;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
import org.eigenbase.rel.FilterRelBase;
import org.eigenbase.rel.RelCollation;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelOptCost;
-import org.eigenbase.relopt.RelOptPlanner;
-import org.eigenbase.relopt.RelTrait;
-import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.relopt.*;
import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.rex.RexBuilder;
-import org.eigenbase.rex.RexCall;
-import org.eigenbase.rex.RexDynamicParam;
-import org.eigenbase.rex.RexInputRef;
-import org.eigenbase.rex.RexLiteral;
-import org.eigenbase.rex.RexLocalRef;
-import org.eigenbase.rex.RexNode;
-import org.eigenbase.rex.RexProgram;
-import org.eigenbase.rex.RexProgramBuilder;
-import org.eigenbase.rex.RexVisitorImpl;
+import org.eigenbase.rex.*;
import org.eigenbase.sql.SqlKind;
import org.eigenbase.sql.SqlOperator;
import org.eigenbase.util.NlsString;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.filter.CaseTupleFilter;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.DynamicTupleFilter;
-import org.apache.kylin.metadata.filter.ExtractTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import java.util.*;
/**
* @author xjiang
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e0bada/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 086d1f1..6a20bdb 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -18,13 +18,6 @@
package org.apache.kylin.query.test;
-import static org.junit.Assert.*;
-
-import java.io.File;
-import java.sql.DriverManager;
-import java.util.List;
-import java.util.Properties;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -40,6 +33,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.File;
+import java.sql.DriverManager;
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertTrue;
+
@Ignore("KylinQueryTest is contained by CombinationTest")
public class KylinQueryTest extends KylinTestBase {
@@ -95,11 +95,13 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql_subquery/query04.sql";
+ String queryFileName = "src/test/resources/query/sample.sql";
File sqlFile = new File(queryFileName);
- runSQL(sqlFile, true, true);
- runSQL(sqlFile, true, false);
+ if (sqlFile.exists()) {
+ runSQL(sqlFile, true, true);
+ runSQL(sqlFile, true, false);
+ }
}
@Test
@@ -208,7 +210,6 @@ public class KylinQueryTest extends KylinTestBase {
}
}
-
@Test
public void testDynamicQuery() throws Exception {
execAndCompDynamicQuery("src/test/resources/query/sql_dynamic", null, true);