You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/13 12:15:47 UTC
[1/2] carbondata git commit: remove column group
Repository: carbondata
Updated Branches:
refs/heads/master a45d68d33 -> ee1e26580
remove column group
fix testcase
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b266f8b3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b266f8b3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b266f8b3
Branch: refs/heads/master
Commit: b266f8b3d496c8d3846313373fb6362dea4cfb88
Parents: a45d68d
Author: jackylk <ja...@huawei.com>
Authored: Tue Jun 13 13:22:01 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 13 17:44:53 2017 +0530
----------------------------------------------------------------------
.../ColumnGroupDataTypesTestCase.scala | 152 -------------------
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 +
.../AutoHighCardinalityIdentifyTestCase.scala | 35 -----
.../org/apache/spark/sql/TableCreator.scala | 4 +
4 files changed, 8 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b266f8b3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
deleted file mode 100644
index ab003c0..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.detailquery
-
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
- * Test Class for aggregate query on multiple datatypes
- *
- */
-class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll {
- sql("drop table if exists colgrp")
- sql("drop table if exists normal")
- sql("drop table if exists colgrp_dictexclude_before")
- sql("drop table if exists colgrp_dictexclude_after")
- sql("drop table if exists colgrp_disorder")
-
- sql("create table colgrp (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
- sql("create table normal (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table normal options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
- //column group with dictionary exclude before column group
- sql("create table colgrp_dictexclude_before (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column1',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp_dictexclude_before options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
- //column group with dictionary exclude after column group
- sql("create table colgrp_dictexclude_after (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_EXCLUDE'='column10',\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp_dictexclude_after options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
-
- }
-
- test("select all dimension query") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp"),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal"))
- }
-
- test("select all dimension query with filter on columnar") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column1='column1666'"),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column1='column1666'"))
- }
-
- test("select all dimension query with filter on column group dimension") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311'"),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311'"))
- }
-
- test("select all dimension query with filter on two dimension from different column group") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column7='column74' "),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column7='column74'"))
- }
-
- test("select all dimension query with filter on two dimension from same column group") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column4='column42' "),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column4='column42'"))
- }
-
- test("select all dimension query with filter on two dimension one from column group other from columnar") {
- checkAnswer(
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from colgrp where column3='column311' and column5='column516' "),
- sql("select column1,column2,column3,column4,column5,column6,column7,column8,column9,column10 from normal where column3='column311' and column5='column516'"))
- }
-
- test("select few dimension") {
- checkAnswer(
- sql("select column1,column3,column4,column5,column6,column9,column10 from colgrp"),
- sql("select column1,column3,column4,column5,column6,column9,column10 from normal"))
- }
-
- test("select count on column group") {
- checkAnswer(
- sql("select count(column2) from colgrp"),
- sql("select count(column2) from normal"))
- }
- test("##ColumnGroup_DictionaryExcludeBefore select all dimension on column group and dictionary exclude table") {
- checkAnswer(
- sql("select * from colgrp_dictexclude_before"),
- sql("select * from normal"))
- }
- test("##ColumnGroup_DictionaryExcludeBefore select all dimension query with filter on two dimension from same column group") {
- checkAnswer(
- sql("select * from colgrp_dictexclude_before where column3='column311' and column4='column42' "),
- sql("select * from normal where column3='column311' and column4='column42'"))
- }
- test("##ColumnGroup_DictionaryExcludeAfter select all dimension on column group and dictionary exclude table") {
- checkAnswer(
- sql("select * from colgrp_dictexclude_after"),
- sql("select * from normal"))
- }
- test("##ColumnGroup_DictionaryExcludeAfter select all dimension query with filter on two dimension from same column group") {
- checkAnswer(
- sql("select * from colgrp_dictexclude_after where column3='column311' and column4='column42' "),
- sql("select * from normal where column3='column311' and column4='column42'"))
- }
- test("ExcludeFilter") {
- checkAnswer(
- sql("select * from colgrp where column3 != 'column311'"),
- sql("select * from normal where column3 != 'column311'"))
-
- checkAnswer(
- sql("select * from colgrp where column3 like 'column31%'"),
- sql("select * from normal where column3 like 'column31%'"))
- checkAnswer(
- sql("select * from colgrp where column3 not like 'column31%'"),
- sql("select * from normal where column3 not like 'column31%'"))
- }
- test("RowFilter") {
- checkAnswer(
- sql("select * from colgrp where column3 != column4"),
- sql("select * from normal where column3 != column4"))
- }
-
- test("Column Group not in order with schema") {
- //Add column group in order different then schema
- try {
- sql("create table colgrp_disorder (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column7,column8),(column2,column3,column4)\")")
- sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp_disorder options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
- assert(true)
- } catch {
- case ex: Exception => assert(false)
- }
-
- }
- override def afterAll {
- sql("drop table colgrp")
- sql("drop table normal")
- sql("drop table colgrp_dictexclude_before")
- sql("drop table colgrp_dictexclude_after")
- sql("drop table if exists colgrp_disorder")
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b266f8b3/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 9d68cd3..e219f08 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -259,6 +259,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// get column groups configuration from table properties.
val groupCols: Seq[String] = updateColumnGroupsInField(tableProperties,
noDictionaryDims, msrs, dims)
+ if (groupCols != null) {
+ throw new MalformedCarbonCommandException(
+ s"${CarbonCommonConstants.COLUMN_GROUPS} is deprecated")
+ }
// get no inverted index columns from table properties.
val noInvertedIdxCols = extractNoInvertedIndexColumns(fields, tableProperties)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b266f8b3/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
index 8841918..598a90e 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
@@ -65,7 +65,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
override def beforeAll {
buildTestData
buildTable
- buildColGrpHighCardTable
}
override def afterAll {
@@ -103,16 +102,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
}
}
- def buildColGrpHighCardTable() {
- try {
- sql("drop table if exists colgrp_highcard")
- sql("""create table if not exists colgrp_highcard
- (hc1 string, c2 string, c3 int)
- STORED BY 'org.apache.carbondata.format' tblproperties('COLUMN_GROUPS'='(hc1,c2)')""")
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
def relation(tableName: String): CarbonRelation = {
CarbonEnv.get.carbonMetastore
.lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
@@ -156,28 +145,4 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
checkMetaData(oldTable, newTable)
}
- test("skip auto identify high cardinality column for column group") {
- val oldTable = relation("colgrp_highcard").tableMeta.carbonTable
- sql(s"LOAD DATA LOCAL INPATH '$filePath' into table colgrp_highcard")
- val newTable = relation("colgrp_highcard").tableMeta.carbonTable
- sql(s"select hc1 from colgrp_highcard")
-
- // check dictionary file
- val tableIdentifier = new CarbonTableIdentifier(newTable.getDatabaseName,
- newTable.getFactTableName, "1")
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storeLocation,
- tableIdentifier)
- val newHc1 = newTable.getDimensionByName("colgrp_highcard", "hc1")
- val newC2 = newTable.getDimensionByName("colgrp_highcard", "c2")
- val dictFileHc1 = carbonTablePath.getDictionaryFilePath(newHc1.getColumnId)
- val dictFileC2 = carbonTablePath.getDictionaryFilePath(newC2.getColumnId)
- assert(CarbonUtil.isFileExists(dictFileHc1))
- assert(CarbonUtil.isFileExists(dictFileC2))
- // check the meta data
- val hc1 = newTable.getDimensionByName("colgrp_highcard", "hc1")
- val c2 = newTable.getDimensionByName("colgrp_highcard", "c2")
- assert(hc1.hasEncoding(Encoding.DICTIONARY))
- assert(c2.hasEncoding(Encoding.DICTIONARY))
-
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b266f8b3/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index 3aa2d4e..3dd05b9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -480,6 +480,10 @@ object TableCreator {
// get column groups configuration from table properties.
val groupCols: Seq[String] = updateColumnGroupsInField(tableProperties,
noDictionaryDims, msrs, dims)
+ if (groupCols != null) {
+ throw new MalformedCarbonCommandException(
+ s"${CarbonCommonConstants.COLUMN_GROUPS} is deprecated")
+ }
// get no inverted index columns from table properties.
val noInvertedIdxCols = extractNoInvertedIndexColumns(fields, tableProperties)
[2/2] carbondata git commit: [CARBONDATA-1064] Make column group
feature deprecated. This closes #1024
Posted by ra...@apache.org.
[CARBONDATA-1064] Make column group feature deprecated. This closes #1024
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee1e2658
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee1e2658
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee1e2658
Branch: refs/heads/master
Commit: ee1e265801b169d42237c52b245ea55d3cfc21b9
Parents: a45d68d b266f8b
Author: ravipesala <ra...@gmail.com>
Authored: Tue Jun 13 17:45:29 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 13 17:45:29 2017 +0530
----------------------------------------------------------------------
.../ColumnGroupDataTypesTestCase.scala | 152 -------------------
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 4 +
.../AutoHighCardinalityIdentifyTestCase.scala | 35 -----
.../org/apache/spark/sql/TableCreator.scala | 4 +
4 files changed, 8 insertions(+), 187 deletions(-)
----------------------------------------------------------------------