You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:05:05 UTC
[07/47] incubator-carbondata git commit: Struct of array is failing
because of complex dimension ordinal (#848)
Struct of array is failing because of complex dimension ordinal (#848)
Problem:
Assignment of complex dimension ordinal was not correct as it was not increment and two children was getting same complex dimension ordinal
Second issue was selecting all the block indexes of complex dimension children.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/07116d24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/07116d24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/07116d24
Branch: refs/heads/master
Commit: 07116d242db652ad8edf6e80526f6a377295343d
Parents: 4cff504
Author: Kumar Vishal <ku...@gmail.com>
Authored: Fri Jul 22 21:13:31 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Fri Jul 22 21:13:31 2016 +0530
----------------------------------------------------------------------
.../metadata/schema/table/CarbonTable.java | 48 ++++++++++++++------
.../schema/table/column/CarbonDimension.java | 4 ++
.../query/carbon/executor/util/QueryUtil.java | 34 +++++++-------
.../spark/src/test/resources/struct_all.csv | 4 ++
.../src/test/resources/structusingstruct.csv | 2 +
.../complexType/TestComplexTypeQuery.scala | 33 +++++++++++++-
6 files changed, 91 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index 6023e7a..4d16659 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -121,23 +121,20 @@ public class CarbonTable implements Serializable {
int columnGroupOrdinal = -1;
int previousColumnGroupId = -1;
List<ColumnSchema> listOfColumns = tableSchema.getListOfColumns();
- int complexTypeOrdinal = 0;
- ;
+ int complexTypeOrdinal = -1;
for (int i = 0; i < listOfColumns.size(); i++) {
ColumnSchema columnSchema = listOfColumns.get(i);
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexTypeOrdinal++);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
dimensions.add(complexDimension);
- int lastDimensionOrdianl=dimensionOrdinal;
dimensionOrdinal =
readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
- listOfColumns, complexDimension, complexTypeOrdinal);
+ listOfColumns, complexDimension);
i = dimensionOrdinal - 1;
- int incrementInOrdinal=dimensionOrdinal-lastDimensionOrdianl;
- complexTypeOrdinal+=incrementInOrdinal;
+ complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
@@ -171,24 +168,21 @@ public class CarbonTable implements Serializable {
* @return
*/
private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount,
- List<ColumnSchema> listOfColumns, CarbonDimension parentDimension,
- int complexDimensionOrdianl) {
+ List<ColumnSchema> listOfColumns, CarbonDimension parentDimension) {
for (int i = 0; i < childCount; i++) {
ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdianl++);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
- listOfColumns, complexDimension, complexDimensionOrdianl);
+ listOfColumns, complexDimension);
} else {
- parentDimension.getListOfChildDimensions().add(
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdianl++));
+ parentDimension.getListOfChildDimensions()
+ .add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
}
}
}
@@ -196,6 +190,30 @@ public class CarbonTable implements Serializable {
}
/**
+ * Read all primitive/complex children and set it as list of child carbon dimension to parent
+ * dimension
+ *
+ * @param dimensionOrdinal
+ * @param childCount
+ * @param listOfColumns
+ * @param parentDimension
+ * @return
+ */
+ private int assignComplexOrdinal(CarbonDimension parentDimension, int complexDimensionOrdianl) {
+ for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
+ CarbonDimension dimension = parentDimension.getListOfChildDimensions().get(i);
+ if (dimension.getNumberOfChild() > 0) {
+ dimension.setComplexTypeOridnal(++complexDimensionOrdianl);
+ complexDimensionOrdianl = assignComplexOrdinal(dimension, complexDimensionOrdianl);
+ } else {
+ parentDimension.getListOfChildDimensions().get(i)
+ .setComplexTypeOridnal(++complexDimensionOrdianl);
+ }
+ }
+ return complexDimensionOrdianl;
+ }
+
+ /**
* @return the databaseName
*/
public String getDatabaseName() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
index 3a4dd11..d3df181 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
@@ -114,6 +114,10 @@ public class CarbonDimension extends CarbonColumn {
return complexTypeOrdinal;
}
+ public void setComplexTypeOridnal(int complexTypeOrdinal) {
+ this.complexTypeOrdinal = complexTypeOrdinal;
+ }
+
/**
* to generate the hash code for this class
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index 241f5b4..2433cbc 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -252,19 +252,22 @@ public class QueryUtil {
blockIndex =
dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ }
}
for (int i = 0; i < dimAggInfo.size(); i++) {
blockIndex = dimensionOrdinalToBlockMapping.get(dimAggInfo.get(i).getDim().getOrdinal());
+ // not adding the children dimension as dimension aggregation
+ // is not push down in case of complex dimension
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, dimAggInfo.get(i).getDim());
-
}
for (int i = 0; i < customAggregationDimension.size(); i++) {
blockIndex =
dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+ // not adding the children dimension as dimension aggregation
+ // is not push down in case of complex dimension
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, customAggregationDimension.get(i));
}
return ArrayUtils
.toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
@@ -274,16 +277,13 @@ public class QueryUtil {
* Below method will be used to add the children block index
* this will be basically for complex dimension which will have children
*
- * @param startBlockIndex start block index
- * @param blockIndexList block index list
- * @param dimension parent dimension
+ * @param blockIndexes block indexes
+ * @param dimension parent dimension
*/
- private static void addChildrenBlockIndex(int startBlockIndex, Set<Integer> blockIndexList,
- CarbonDimension dimension) {
+ private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) {
for (int i = 0; i < dimension.numberOfChild(); i++) {
- blockIndexList.add(++startBlockIndex);
- addChildrenBlockIndex(startBlockIndex, blockIndexList,
- dimension.getListOfChildDimensions().get(i));
+ addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
+ blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
}
}
@@ -379,7 +379,7 @@ public class QueryUtil {
if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
dictionaryDimensionFromQuery);
- } else if(!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
dictionaryDimensionFromQuery
.add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
}
@@ -873,7 +873,7 @@ public class QueryUtil {
&& queryDimension.getDimension().numberOfChild() == 0) {
dictionaryDimensionBlockIndex
.add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- } else if(queryDimension.getDimension().numberOfChild() == 0){
+ } else if (queryDimension.getDimension().numberOfChild() == 0) {
noDictionaryDimensionBlockIndex
.add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
}
@@ -1078,9 +1078,9 @@ public class QueryUtil {
dimension.getColName(), ++parentBlockIndex));
break;
default:
- boolean isDirectDictionary = CarbonUtil.hasEncoding(
- dimension.getListOfChildDimensions().get(i).getEncoder(),
- Encoding.DIRECT_DICTIONARY);
+ boolean isDirectDictionary = CarbonUtil
+ .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
+ Encoding.DIRECT_DICTIONARY);
parentQueryType.addChildren(
new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
dimension.getColName(), ++parentBlockIndex,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/struct_all.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/struct_all.csv b/integration/spark/src/test/resources/struct_all.csv
new file mode 100644
index 0000000..02b6387
--- /dev/null
+++ b/integration/spark/src/test/resources/struct_all.csv
@@ -0,0 +1,4 @@
+10&10$10&10
+20&20$20&20
+30&30$30&30
+40&40$40&40
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/structusingstruct.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/structusingstruct.csv b/integration/spark/src/test/resources/structusingstruct.csv
new file mode 100644
index 0000000..5a2bae1
--- /dev/null
+++ b/integration/spark/src/test/resources/structusingstruct.csv
@@ -0,0 +1,2 @@
+def$klm&abc$12
+pri$sac&pra$18
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index d17840f..ab48133 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -37,12 +37,37 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
sql("drop table if exists complexcarbontable").show
sql("drop table if exists complexhivetable").show
+ sql("drop table if exists structusingstructCarbon").show
+ sql("drop table if exists structusingstructHive").show
+ sql("drop table if exists structusingarraycarbon").show
+ sql("drop table if exists structusingarrayhive").show
sql("create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId', 'DICTIONARY_EXCLUDE'='channelsId','COLUMN_GROUP'='(ROMSize,ROMName)')");
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexcarbontable OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', 'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
sql("create table complexhivetable(deviceInformationId int, channelsId string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId double,contractNumber double)row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by ':'")
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexhivetable");
+
+ sql("create table structusingarraycarbon (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>) STORED BY 'org.apache.carbondata.format'");
+ sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarraycarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+ sql("create table structusingarrayhive (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>)row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by '&'");
+ sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarrayhive")
+
+ sql("create table structusingstructCarbon(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int> ) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table structusingstructCarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='name','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+ sql("create table structusingstructhive(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int> )row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by '&'")
+ sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table structusingstructhive")
+
}
+ test("select * from structusingarraycarbon") {
+ checkAnswer(sql("select * from structusingarraycarbon"),
+ sql("select * from structusingarrayhive"))
+ }
+
+ test("select * from structusingstructCarbon") {
+ checkAnswer(sql("select * from structusingstructCarbon"),
+ sql("select * from structusingstructhive"))
+ }
+
test("select * from complexcarbontable") {
checkAnswer(sql("select * from complexcarbontable"),
sql("select * from complexhivetable"))
@@ -112,8 +137,12 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("drop table if exists complexcarbontable")
- sql("drop table if exists complexhivetable")
+ sql("drop table if exists complexcarbontable").show
+ sql("drop table if exists complexhivetable").show
+ sql("drop table if exists structusingstructCarbon").show
+ sql("drop table if exists structusingstructHive").show
+ sql("drop table if exists structusingarraycarbon").show
+ sql("drop table if exists structusingarrayhive").show
}
}