You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/07/20 10:13:57 UTC
[29/50] [abbrv] incubator-carbondata git commit: [Bug] While defining
Column groups,
if group is not defined in schema order than data loading is failing (#829)
[Bug] While defining Column groups, if group is not defined in schema order than data loading is failing (#829)
This fix is to arrange the group in schema order
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8b3c1350
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8b3c1350
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8b3c1350
Branch: refs/heads/master
Commit: 8b3c13506e3344f01b2893704baf411ec76501f9
Parents: 8c83954
Author: ashokblend <as...@gmail.com>
Authored: Mon Jul 18 16:51:30 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 18 16:51:30 2016 +0530
----------------------------------------------------------------------
.../CarbonDictionaryMetadataReaderImpl.java | 8 +++--
.../core/reader/CarbonDictionaryReaderImpl.java | 8 +++--
.../CarbonDictionarySortIndexReaderImpl.java | 8 +++--
.../CarbonDictionarySortIndexWriterImpl.java | 8 +++--
.../org/apache/spark/sql/CarbonSqlParser.scala | 2 +-
.../org/carbondata/spark/util/CommonUtil.scala | 32 ++++++++++++++++++++
.../apache/spark/sql/TestCarbonSqlParser.scala | 11 +++++++
.../ColumnGroupDataTypesTestCase.scala | 13 ++++++++
8 files changed, 77 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index a43fc22..227dc8d 100644
--- a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -23,11 +23,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.carbondata.common.factory.CarbonCommonFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.service.PathService;
import org.carbondata.format.ColumnDictionaryChunkMeta;
import org.apache.thrift.TBase;
@@ -154,8 +155,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
* This method will form the path for dictionary metadata file for a given column
*/
protected void initFileLocation() {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(this.hdfsStorePath, carbonTableIdentifier);
+ PathService pathService = CarbonCommonFactory.getPathService();
+ CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(columnIdentifier,
+ this.hdfsStorePath, carbonTableIdentifier);
this.columnDictionaryMetadataFilePath =
carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index 5e5207a..a93d47a 100644
--- a/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -24,11 +24,12 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.carbondata.common.factory.CarbonCommonFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.service.PathService;
import org.carbondata.format.ColumnDictionaryChunk;
import org.apache.thrift.TBase;
@@ -199,8 +200,9 @@ public class CarbonDictionaryReaderImpl implements CarbonDictionaryReader {
* This method will form the path for dictionary file for a given column
*/
protected void initFileLocation() {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(this.hdfsStorePath, carbonTableIdentifier);
+ PathService pathService = CarbonCommonFactory.getPathService();
+ CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(columnIdentifier,
+ this.hdfsStorePath, carbonTableIdentifier);
this.columnDictionaryFilePath = carbonTablePath
.getDictionaryFilePath(columnIdentifier.getColumnId());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 337a857..54e8950 100644
--- a/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -21,14 +21,15 @@ package org.carbondata.core.reader.sortindex;
import java.io.IOException;
import java.util.List;
+import org.carbondata.common.factory.CarbonCommonFactory;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonStorePath;
import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.datastorage.store.impl.FileFactory;
import org.carbondata.core.reader.ThriftReader;
+import org.carbondata.core.service.PathService;
import org.carbondata.core.util.CarbonUtil;
import org.carbondata.format.ColumnSortInfo;
@@ -150,8 +151,9 @@ public class CarbonDictionarySortIndexReaderImpl implements CarbonDictionarySort
}
protected void initPath() {
- CarbonTablePath carbonTablePath =
- CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier);
+ PathService pathService = CarbonCommonFactory.getPathService();
+ CarbonTablePath carbonTablePath = pathService
+ .getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
this.sortIndexFilePath =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 89fba44..f0b935c 100644
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
-import org.carbondata.common.ext.PathFactory;
+import org.carbondata.common.factory.CarbonCommonFactory;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
@@ -32,6 +32,7 @@ import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.service.PathService;
import org.carbondata.core.util.CarbonProperties;
import org.carbondata.core.util.CarbonUtil;
import org.carbondata.core.writer.ThriftWriter;
@@ -150,7 +151,8 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
}
protected void initPath() {
- CarbonTablePath carbonTablePath = PathFactory.getInstance()
+ PathService pathService = CarbonCommonFactory.getPathService();
+ CarbonTablePath carbonTablePath = pathService
.getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
@@ -164,7 +166,7 @@ public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySort
*
* @param carbonTablePath
*/
- private void cleanUpOldSortIndex(CarbonTablePath carbonTablePath) {
+ protected void cleanUpOldSortIndex(CarbonTablePath carbonTablePath) {
CarbonFile sortIndexFile =
FileFactory.getCarbonFile(sortIndexFilePath, FileFactory.getFileType(sortIndexFilePath));
CarbonFile[] files =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 7de452a..40f9e70 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -527,7 +527,7 @@ class CarbonSqlParser()
splittedColGrps :+= arrangedColGrp
}
// This will be furthur handled.
- splittedColGrps
+ CommonUtil.arrangeColGrpsInSchemaOrder(splittedColGrps, dims)
}
else {
null
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
index 5b8c0b2..708a00b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CommonUtil.scala
@@ -167,4 +167,36 @@ object CommonUtil {
false
}
}
+
+ /**
+ * @param colGrps
+ * @param dims
+ * @return columns of column groups in schema order
+ */
+ def arrangeColGrpsInSchemaOrder(colGrps: Seq[String], dims: Seq[Field]): Seq[String] = {
+ def sortByIndex(colGrp1: String, colGrp2: String) = {
+ val firstCol1 = colGrp1.split(",")(0)
+ val firstCol2 = colGrp2.split(",")(0)
+ val dimIndex1: Int = getDimIndex(firstCol1, dims)
+ val dimIndex2: Int = getDimIndex(firstCol2, dims)
+ dimIndex1 < dimIndex2
+ }
+ val sortedColGroups: Seq[String] = colGrps.sortWith(sortByIndex)
+ sortedColGroups
+ }
+
+ /**
+ * @param colName
+ * @param dims
+ * @return return index for given column in dims
+ */
+ def getDimIndex(colName: String, dims: Seq[Field]): Int = {
+ var index: Int = -1
+ dims.zipWithIndex.foreach { h =>
+ if (h._1.column.equalsIgnoreCase(colName)) {
+ index = h._2.toInt
+ }
+ }
+ index
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index f825ce2..983d488 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -79,6 +79,17 @@ class TestCarbonSqlParser extends QueryTest {
assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
}
+ test("Test-updateColumnGroupsInField_disordered") {
+ val colGroupStr = "(col5,col6),(col2,col3),(col7,col8)"
+ val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
+ var fields: Seq[Field] = loadAllFields
+ val stub = new TestCarbonSqlParserStub()
+ val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
+ assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
+ assert(colgrps.lift(1).get.equalsIgnoreCase("col5,col6"))
+ assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
+
+ }
test("Test-ColumnGroupsInvalidField_Shouldnotallow") {
val colGroupStr = "(col1,col2),(col10,col6),(col7,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8b3c1350/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
index 122e0f1..759b84c 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
@@ -40,6 +40,7 @@ class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
//column group with dictionary exclude after column group
sql("create table colgrp_dictexclude_after (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column10',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_dictexclude_after options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
+
}
test("select all dimension query") {
@@ -127,10 +128,22 @@ class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
sql("select * from normal where column3 != column4"))
}
+ test("Column Group not in order with schema") {
+ //Add column group in order different then schema
+ try {
+ sql("create table colgrp_disorder (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column7,column8),(column2,column3,column4)\")")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/10dim_4msr.csv' INTO table colgrp_disorder options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
+ assert(true)
+ } catch {
+ case ex: Exception => assert(false)
+ }
+
+ }
override def afterAll {
sql("drop table colgrp")
sql("drop table normal")
sql("drop table colgrp_dictexclude_before")
sql("drop table colgrp_dictexclude_after")
+ sql("drop table if exists colgrp_disorder")
}
}
\ No newline at end of file