You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/06/19 06:29:19 UTC
[1/2] kylin git commit: KYLIN-1766 Alter LookupTable to hold millis
instead of yyyymmdd
Repository: kylin
Updated Branches:
refs/heads/master a82662bcd -> 11e20adbd
KYLIN-1766 Alter LookupTable to hold millis instead of yyyymmdd
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a96b57f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a96b57f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a96b57f
Branch: refs/heads/master
Commit: 8a96b57f066ed58d55339d3b8e18bbf51c45ec34
Parents: a82662b
Author: Yang Li <li...@apache.org>
Authored: Sat Jun 18 22:34:04 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jun 19 14:15:51 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 2 -
.../java/org/apache/kylin/cube/CubeSegment.java | 3 +-
.../apache/kylin/cube/kv/RowKeyColumnOrder.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 4 +-
.../kylin/gridtable/GTScanRangePlanner.java | 1 -
.../kylin/dict/lookup/LookupBytesTable.java | 56 ---------------
.../kylin/dict/lookup/LookupStringTable.java | 71 +++++++++++++++----
.../apache/kylin/dict/lookup/LookupTable.java | 18 +++--
.../kylin/dict/lookup/LookupTableTest.java | 19 +++--
.../metadata/filter/ConstantTupleFilter.java | 5 +-
.../apache/kylin/engine/mr/LookupTableTest.java | 74 --------------------
.../resources/query/sql_derived/query11.sql | 26 +++++++
.../kylin/query/enumerator/OLAPEnumerator.java | 25 ++-----
.../kylin/query/relnode/OLAPFilterRel.java | 25 ++++++-
storage-hbase/pom.xml | 4 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +-
16 files changed, 157 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a33cf00..caa2765 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -201,8 +201,6 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo info = null;
try {
DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
String dictResPath = cubeSeg.getDictResPath(col);
if (dictResPath == null)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index f79e06d..b9195fd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -305,7 +305,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
public Dictionary<String> getDictionary(TblColRef col) {
TblColRef reuseCol = getCubeDesc().getDictionaryReuseColumn(col);
- return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, reuseCol);
+ CubeManager cubeMgr = CubeManager.getInstance(this.getCubeInstance().getConfig());
+ return cubeMgr.getDictionary(this, reuseCol);
}
public CubeDimEncMap getDimensionEncodingMap() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
index fea3736..23c8f6a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
@@ -32,7 +32,7 @@ abstract public class RowKeyColumnOrder implements Comparator<String> {
public static final StringOrder STRING_ORDER = new StringOrder();
public static RowKeyColumnOrder getInstance(DataType type) {
- if (type.isNumberFamily())
+ if (type.isNumberFamily() || type.isDateTimeFamily())
return NUMBER_ORDER;
else
return STRING_ORDER;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index c4174a6..d4cfcb7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -79,10 +79,10 @@ public class RowKeyColDesc {
if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) {
DataType type = colRef.getType();
if (type.isDate()) {
- encodingName = DateDimEnc.ENCODING_NAME;
+ encoding = encodingName = DateDimEnc.ENCODING_NAME;
}
if (type.isTime() || type.isTimestamp() || type.isDatetime()) {
- encodingName = TimeDimEnc.ENCODING_NAME;
+ encoding = encodingName = TimeDimEnc.ENCODING_NAME;
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index baf3428..7d31bf7 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -156,7 +156,6 @@ public class GTScanRangePlanner {
//start key GTRecord compare to stop key GTRecord
this.rangeStartEndComparator = getRangeStartEndComparator(comp);
-
this.gtFilter = gtFilter;
this.gtStartAndEnd = gtStartAndEnd;
this.gtPartitionCol = gtPartitionCol;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
deleted file mode 100644
index 0758edc..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ReadableTable;
-
-/**
- * @author yangli9
- *
- */
-public class LookupBytesTable extends LookupTable<ByteArray> {
-
- public LookupBytesTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
- super(tableDesc, keyColumns, table);
- }
-
- @Override
- protected ByteArray[] convertRow(String[] cols) {
- ByteArray[] r = new ByteArray[cols.length];
- for (int i = 0; i < cols.length; i++) {
- r[i] = cols[i] == null ? null : new ByteArray(Bytes.toBytes(cols[i]));
- }
- return r;
- }
-
- @Override
- protected String toString(ByteArray cell) {
- return cell.toString();
- }
-
- public Class<?> getType() {
- return ByteArray.class;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
index 9f6346a..fd88fb0 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
@@ -19,8 +19,10 @@
package org.apache.kylin.dict.lookup;
import java.io.IOException;
+import java.util.Comparator;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ReadableTable;
@@ -30,37 +32,80 @@ import org.apache.kylin.source.ReadableTable;
*
*/
public class LookupStringTable extends LookupTable<String> {
+
+ private static final Comparator<String> dateStrComparator = new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ long l1 = Long.parseLong(o1);
+ long l2 = Long.parseLong(o2);
+ return Long.compare(l1, l2);
+ }
+ };
- int[] keyIndexOfDates;
+ private static final Comparator<String> numStrComparator = new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ double d1 = Double.parseDouble(o1);
+ double d2 = Double.parseDouble(o2);
+ return Double.compare(d1, d2);
+ }
+ };
+
+ private static final Comparator<String> defaultStrComparator = new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ return o1.compareTo(o2);
+ }
+ };
+
+ boolean[] colIsDateTime;
+ boolean[] colIsNumber;
public LookupStringTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table) throws IOException {
super(tableDesc, keyColumns, table);
}
+
+ @Override
+ protected void init() throws IOException {
+ ColumnDesc[] cols = tableDesc.getColumns();
+ colIsDateTime = new boolean[cols.length];
+ colIsNumber = new boolean[cols.length];
+ for (int i = 0; i < cols.length; i++) {
+ DataType t = cols[i].getType();
+ colIsDateTime[i] = t.isDateTimeFamily();
+ colIsNumber[i] = t.isNumberFamily();
+ }
+
+ super.init();
+ }
@Override
protected String[] convertRow(String[] cols) {
- if (keyIndexOfDates == null) {
- keyIndexOfDates = new int[keyColumns.length];
- for (int i = 0; i < keyColumns.length; i++) {
- ColumnDesc col = tableDesc.findColumnByName(keyColumns[i]);
- keyIndexOfDates[i] = col.getType().isDateTimeFamily() ? col.getZeroBasedIndex() : -1;
+ for (int i = 0; i < cols.length; i++) {
+ if (colIsDateTime[i]) {
+ cols[i] = String.valueOf(DateFormat.stringToMillis(cols[i]));
}
}
-
- for (int i = 0; i < keyIndexOfDates.length; i++) {
- int c = keyIndexOfDates[i];
- if (c >= 0)
- cols[c] = String.valueOf(DateFormat.stringToMillis(cols[c]));
- }
return cols;
}
@Override
+ protected Comparator<String> getComparator(int idx) {
+ if (colIsDateTime[idx])
+ return dateStrComparator;
+ else if (colIsNumber[idx])
+ return numStrComparator;
+ else
+ return defaultStrComparator;
+ }
+
+ @Override
protected String toString(String cell) {
return cell;
}
public Class<?> getType() {
return String.class;
- }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 9abd28c..5a96df6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -39,7 +40,7 @@ import com.google.common.collect.Sets;
*
* @author yangli9
*/
-abstract public class LookupTable<T extends Comparable<T>> {
+abstract public class LookupTable<T> {
protected TableDesc tableDesc;
protected String[] keyColumns;
@@ -109,15 +110,18 @@ abstract public class LookupTable<T extends Comparable<T>> {
public Pair<T, T> mapRange(String col, T beginValue, T endValue, String returnCol) {
int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
+ Comparator<T> colComp = getComparator(colIdx);
+ Comparator<T> returnComp = getComparator(returnIdx);
+
T returnBegin = null;
T returnEnd = null;
for (T[] row : data.values()) {
- if (between(beginValue, row[colIdx], endValue)) {
+ if (between(beginValue, row[colIdx], endValue, colComp)) {
T returnValue = row[returnIdx];
- if (returnBegin == null || returnValue.compareTo(returnBegin) < 0) {
+ if (returnBegin == null || returnComp.compare(returnValue, returnBegin) < 0) {
returnBegin = returnValue;
}
- if (returnEnd == null || returnValue.compareTo(returnEnd) > 0) {
+ if (returnEnd == null || returnComp.compare(returnValue, returnEnd) > 0) {
returnEnd = returnValue;
}
}
@@ -140,10 +144,12 @@ abstract public class LookupTable<T extends Comparable<T>> {
return result;
}
- private boolean between(T beginValue, T v, T endValue) {
- return (beginValue == null || beginValue.compareTo(v) <= 0) && (endValue == null || v.compareTo(endValue) <= 0);
+ private boolean between(T beginValue, T v, T endValue, Comparator<T> comp) {
+ return (beginValue == null || comp.compare(beginValue, v) <= 0) && (endValue == null || comp.compare(v, endValue) <= 0);
}
+ abstract protected Comparator<T> getComparator(int colIdx);
+
public String toString() {
return "LookupTable [path=" + table + "]";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 980a263..3f67dcd 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -65,7 +65,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
for (String i : results) {
System.out.println(i);
- Assert.assertEquals("2012-01-01", i);
+ Assert.assertEquals(millis("2012-01-01"), i);
}
}
@@ -77,11 +77,22 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
System.out.println("The first qtr_beg_dt is " + results.getFirst());
System.out.println("The last qtr_beg_dt is " + results.getSecond());
- Assert.assertEquals("2012-01-01", results.getFirst());
- Assert.assertEquals("2012-10-01", results.getSecond());
+ Assert.assertEquals(millis("2012-01-01"), results.getFirst());
+ Assert.assertEquals(millis("2012-10-01"), results.getSecond());
}
@Test
+ public void testMapRange2() throws Exception {
+ Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"), "CAL_DT");
+
+ System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getFirst())));
+ System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getSecond())));
+
+ Assert.assertEquals(millis("2013-05-05"), results.getFirst());
+ Assert.assertEquals(millis("2013-08-03"), results.getSecond());
+ }
+
+ @Test
public void testMapValues() throws Exception {
Set<String> values = new HashSet<String>();
values.add(millis("2012-01-24"));
@@ -92,7 +103,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
for (String i : results) {
System.out.println(i);
- Assert.assertEquals("2012-01-01", i);
+ Assert.assertEquals(millis("2012-01-01"), i);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index db3eb4f..61d87e8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -21,13 +21,12 @@ package org.apache.kylin.metadata.filter;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
-import java.util.TreeSet;
-import com.google.common.collect.Lists;
-import org.apache.commons.collections.comparators.NullComparator;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import com.google.common.collect.Lists;
+
/**
*
* @author xjiang
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
deleted file mode 100644
index 87f9133..0000000
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import java.io.File;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.lookup.LookupBytesTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setup() throws Exception {
- createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- cleanupTestMetadata();
- }
-
- @Test
- public void testBasic() throws Exception {
- TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES");
- TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings");
- LookupBytesTable lookup;
-
- System.out.println("============================================================================");
-
- File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv");
- lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new DFSFileTable("file://" + f.getAbsolutePath(), 10));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv");
- lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id" }, new DFSFileTable("file://" + f.getAbsolutePath(), 36));
- lookup.dump();
-
- System.out.println("============================================================================");
-
- ByteArray k1 = new ByteArray(Bytes.toBytes("533"));
- ByteArray k2 = new ByteArray(Bytes.toBytes("0"));
- Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2 });
- System.out.println(lookup.getRow(key));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/kylin-it/src/test/resources/query/sql_derived/query11.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query11.sql b/kylin-it/src/test/resources/query/sql_derived/query11.sql
new file mode 100644
index 0000000..41157c5
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_derived/query11.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name, sum(test_kylin_fact.price) as gmv, count(*) as trans_cnt
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-10-01'
+ group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 07f72b1..ef4ddf8 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -24,11 +24,9 @@ import java.util.Properties;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
-import org.apache.kylin.dict.DictCodeSystem;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.TimeConditionLiteralsReplacer;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
@@ -112,11 +110,8 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// bind dynamic variables
bindVariable(olapContext.filter);
- // modify date condition
- olapContext.filter = modifyTimeLiterals(olapContext.filter);
- olapContext.resetSQLDigest();
-
// cube don't have correct result for simple query without group by, but let's try to return something makes sense
+ olapContext.resetSQLDigest();
SQLDigest sqlDigest = olapContext.getSQLDigest();
hackNoGroupByAggregation(sqlDigest);
@@ -130,16 +125,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
return iterator;
}
- /**
- * Calcite passed down all time family constants as GregorianCalendar,
- * we'll have to reformat it to date/time according to column definition
- */
- private TupleFilter modifyTimeLiterals(TupleFilter filter) {
- TimeConditionLiteralsReplacer filterDecorator = new TimeConditionLiteralsReplacer(filter);
- byte[] bytes = TupleFilterSerializer.serialize(filter, filterDecorator, DictCodeSystem.INSTANCE);
- return TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
- }
-
private void bindVariable(TupleFilter filter) {
if (filter == null) {
return;
@@ -155,7 +140,11 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
String variable = entry.getKey();
Object value = optiqContext.get(variable);
if (value != null) {
- compFilter.bindVariable(variable, value.toString());
+ String str = value.toString();
+ if (compFilter.getColumn().getType().isDateTimeFamily())
+ str = String.valueOf(DateFormat.stringToMillis(str));
+
+ compFilter.bindVariable(variable, str);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 2806b4b..ae6b265 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -50,7 +50,9 @@ import org.apache.calcite.rex.RexProgramBuilder;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.util.NlsString;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.metadata.filter.CaseTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -64,6 +66,7 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
/**
@@ -144,7 +147,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
for (RexNode operand : call.operands) {
TupleFilter childFilter = operand.accept(this);
if (filter == null) {
- filter = childFilter;
+ filter = cast(childFilter, call.type);
} else {
filter.addChild(childFilter);
}
@@ -159,6 +162,26 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
return filter;
}
+ private TupleFilter cast(TupleFilter filter, RelDataType type) {
+ if ((filter instanceof ConstantTupleFilter) == false) {
+ return filter;
+ }
+
+ ConstantTupleFilter constFilter = (ConstantTupleFilter) filter;
+
+ if (type.getFamily() == SqlTypeFamily.DATE || type.getFamily() == SqlTypeFamily.DATETIME || type.getFamily() == SqlTypeFamily.TIMESTAMP) {
+ List<String> newValues = Lists.newArrayList();
+ for (Object v : constFilter.getValues()) {
+ if (v == null)
+ newValues.add(null);
+ else
+ newValues.add(String.valueOf(DateFormat.stringToMillis(v.toString())));
+ }
+ constFilter = new ConstantTupleFilter(newValues);
+ }
+ return constFilter;
+ }
+
private CompareTupleFilter mergeToInClause(TupleFilter filter) {
List<? extends TupleFilter> children = filter.getChildren();
TblColRef inColumn = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 6c9e284..943b2d4 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -137,8 +137,10 @@
<include>org.apache.kylin:kylin-core-cube</include>
<include>org.apache.kylin:kylin-invertedindex</include>
<include>com.ning:compress-lzf</include>
- <include>com.n3twork.druid:extendedset</include>
<include>org.roaringbitmap:RoaringBitmap</include>
+ <!-- below for inverted index only -->
+ <include>com.n3twork.druid:extendedset</include>
+ <include>org.apache.commons:commons-lang3</include>
</includes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4b3cb67..672bcbe 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -48,6 +48,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
+import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -543,10 +544,13 @@ public class CubeStorageQuery implements IStorageQuery {
}
// a little pre-evaluation to remove invalid EQ/IN values and round start/end according to dictionary
+ RowKeyDesc rowkey = cubeSegment.getCubeDesc().getRowkey();
Iterator<ColumnValueRange> it = rangeMap.values().iterator();
while (it.hasNext()) {
ColumnValueRange range = it.next();
- range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn()));
+ if (rowkey.isUseDictionary(range.getColumn())) {
+ range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn()));
+ }
if (range.satisfyAll())
it.remove();
else if (range.satisfyNone())
[2/2] kylin git commit: disable ITIIQueryTest temporarily
Posted by li...@apache.org.
disable ITIIQueryTest temporarily
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11e20adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11e20adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11e20adb
Branch: refs/heads/master
Commit: 11e20adbdfc1ed2de34280d50b90176f59d1760a
Parents: 8a96b57
Author: Yang Li <li...@apache.org>
Authored: Sun Jun 19 14:29:01 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jun 19 14:29:01 2016 +0800
----------------------------------------------------------------------
kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/11e20adb/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
index 985424c..09253c3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java
@@ -35,7 +35,8 @@ import com.google.common.collect.Maps;
/**
*/
-@RunWith(Parameterized.class)
+@Ignore("II query is no longer maintained")
+//@RunWith(Parameterized.class)
public class ITIIQueryTest extends ITKylinQueryTest {
@BeforeClass
public static void setUp() throws Exception {