You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/24 13:26:39 UTC
[43/50] [abbrv] carbondata git commit: [CARBONDATA-1930] Added
condition to refer to parent dictionary if filter is given in aggregate table
query
[CARBONDATA-1930] Added condition to refer to parent dictionary if filter is given in aggregate table query
When filter is applied to aggregate query then the query is not accessing the parent table dictionary instead it is trying to search for aggregate table dictionary files.Add check to access parent table dictionary files if aggregate column has parent columns.
This closes #1710
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1381c9f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1381c9f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1381c9f1
Branch: refs/heads/fgdatamap
Commit: 1381c9f19508f68f95953c48126e27ca239818d1
Parents: 406d754
Author: kunal642 <ku...@gmail.com>
Authored: Fri Dec 22 13:23:11 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Fri Dec 22 21:41:31 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/scan/filter/FilterUtil.java | 16 +++++++++++++++-
.../TestPreAggregateTableSelection.scala | 11 +++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index a8fa8ec..3268ca3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -61,6 +62,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -1241,15 +1243,27 @@ public final class FilterUtil {
AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier,
CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException {
String dictionaryPath = null;
+ ColumnIdentifier columnIdentifier = carbonDimension.getColumnIdentifier();
if (null != tableProvider) {
CarbonTable carbonTable = tableProvider
.getCarbonTable(dictionarySourceAbsoluteTableIdentifier.getCarbonTableIdentifier());
dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties()
.get(CarbonCommonConstants.DICTIONARY_PATH);
+ if (null != carbonDimension.getColumnSchema().getParentColumnTableRelations() &&
+ carbonDimension.getColumnSchema().getParentColumnTableRelations().size() == 1) {
+ dictionarySourceAbsoluteTableIdentifier =
+ QueryUtil.getTableIdentifierForColumn(carbonDimension,
+ carbonTable.getAbsoluteTableIdentifier());
+ columnIdentifier = new ColumnIdentifier(
+ carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(),
+ carbonDimension.getColumnProperties(), carbonDimension.getDataType());
+ } else {
+ dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
+ }
}
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier,
- carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), dictionaryPath);
+ columnIdentifier, carbonDimension.getDataType(), dictionaryPath);
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
cacheProvider.createCache(CacheType.FORWARD_DICTIONARY);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 7157b8a..fdeb2bd 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -219,6 +219,17 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
}
}
+ test("test if pre-agg table is hit with filter condition") {
+ sql("drop table if exists filtertable")
+ sql("CREATE TABLE filtertable(id int, name string, city string, age string) STORED BY" +
+ " 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='name,age')")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table filtertable")
+ sql("create datamap agg9 on table filtertable using 'preaggregate' as select name, age, sum(age) from filtertable group by name, age")
+ val df = sql("select name, sum(age) from filtertable where age = '29' group by name, age")
+ preAggTableValidator(df.queryExecution.analyzed, "filtertable_agg9")
+ checkAnswer(df, Row("vishal", 29))
+ }
+
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")