You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/05/07 05:59:03 UTC
carbondata git commit: [CARBONDATA-2436] Block pruning problem post
the carbon schema restructure.
Repository: carbondata
Updated Branches:
refs/heads/master 729286919 -> a7926ea13
[CARBONDATA-2436] Block pruning problem post the carbon schema restructure.
Currently DataMap is pruning with segmentproperties from the 0th block of BlockletDataMap is not correct. As post restructure if the table is updated then all the block
will not have symmetric schema within the same segments.
Solution : It must be ensured the block could be pruned with the same schema.
This closes #2263
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a7926ea1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a7926ea1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a7926ea1
Branch: refs/heads/master
Commit: a7926ea133170bd027756969410e043eb9a4b7a3
Parents: 7292869
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri May 4 16:36:57 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Mon May 7 11:32:18 2018 +0530
----------------------------------------------------------------------
.../blockletindex/BlockletDataMap.java | 6 ++++-
.../AlterTableValidationTestCase.scala | 23 ++++++++++++++++++--
2 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 906291a..f72dc06 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -685,7 +685,11 @@ public class BlockletDataMap extends CoarseGrainDataMap implements Cacheable {
}
}
// Prune with filters if the partitions are existed in this datamap
- return prune(filterExp, segmentProperties);
+ // changed segmentProperties to this.segmentProperties to make sure the pruning with its own
+ // segmentProperties.
+ // Its a temporary fix. The Interface DataMap.prune(FilterResolverIntf filterExp,
+ // SegmentProperties segmentProperties, List<PartitionSpec> partitions) should be corrected
+ return prune(filterExp, this.segmentProperties);
}
private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index a86bf17..648ed11 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -43,7 +43,7 @@ class AlterTableValidationTestCase extends Spark2QueryTest with BeforeAndAfterAl
sql("drop table if exists testalterwithboolean")
sql("drop table if exists testalterwithbooleanwithoutdefaultvalue")
sql("drop table if exists test")
-
+ sql("drop table if exists retructure_iud")
// clean data folder
CarbonProperties.getInstance()
@@ -690,7 +690,25 @@ test("test alter command for boolean data type with correct default measure valu
testFilterWithDefaultValue(true)
testFilterWithDefaultValue(false)
}
-
+ test("Filter query on Restructure and updated table") {
+ sql(
+ """
+ CREATE TABLE retructure_iud(id int, name string, city string, age int)
+ STORED BY 'org.apache.carbondata.format'
+ """)
+ val testData = s"$resourcesPath/sample.csv"
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table retructure_iud")
+ sql("ALTER TABLE retructure_iud ADD COLUMNS (newField STRING) " +
+ "TBLPROPERTIES ('DEFAULT.VALUE.newField'='def', 'DICTIONARY_INCLUDE'='newField')").show()
+ sql("ALTER TABLE retructure_iud ADD COLUMNS (newField1 STRING) " +
+ "TBLPROPERTIES ('DEFAULT.VALUE.newField1'='def', 'DICTIONARY_EXCLUDE'='newField1')").show()
+ // update operation
+ sql("""update retructure_iud d set (d.id) = (d.id + 1) where d.id > 2""").show()
+ checkAnswer(
+ sql("select count(*) from retructure_iud where id = 2 and newfield1='def'"),
+ Seq(Row(1))
+ )
+ }
override def afterAll {
sql("DROP TABLE IF EXISTS restructure")
sql("drop table if exists table1")
@@ -707,5 +725,6 @@ test("test alter command for boolean data type with correct default measure valu
sql("drop table if exists testalterwithboolean")
sql("drop table if exists testalterwithbooleanwithoutdefaultvalue")
sql("drop table if exists test")
+ sql("drop table if exists retructure_iud")
}
}