You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/11/14 17:50:30 UTC
[08/18] carbondata git commit: [CARBONDATA-1574] No_Inverted is
applied for all newly added column
[CARBONDATA-1574] No_Inverted is applied for all newly added column
If No_Inverted_Index is specified in tableproperties during addition of several column.
All columns are being considered as No_Inverted_Index, even if few of them is marked as No_Inverted_Index in tableproperties .
Solution : Match column name before applying the property.
This closes #1414
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b2b030e5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b2b030e5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b2b030e5
Branch: refs/heads/fgdatamap
Commit: b2b030e5c8576405446f3b784b68c0a1ddd1ea8a
Parents: d2319f0
Author: rahulforallp <ra...@knoldus.in>
Authored: Fri Oct 13 19:18:53 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Nov 14 17:55:48 2017 +0530
----------------------------------------------------------------------
.../execution/command/carbonTableSchemaCommon.scala | 3 ++-
.../org/apache/spark/sql/test/util/QueryTest.scala | 13 +++++++++++++
.../vectorreader/AddColumnTestCases.scala | 15 +++++++++++++++
3 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2b030e5/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 759d3d8..a2f3364 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -304,7 +304,8 @@ class AlterTableColumnSchemaGenerator(
}
}
}
- else if (elem._1.equalsIgnoreCase("no_inverted_index")) {
+ else if (elem._1.equalsIgnoreCase("no_inverted_index") &&
+ (elem._2.split(",").contains(col.getColumnName))) {
col.getEncodingList.remove(Encoding.INVERTED_INDEX)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2b030e5/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
index d96ef47..0a4231c 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
@@ -30,6 +30,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+
+
class QueryTest extends PlanTest {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
@@ -58,6 +60,17 @@ class QueryTest extends PlanTest {
}
}
+ /**
+ * Runs the plan and counts the keyword in the answer
+ * @param df the [[DataFrame]] to be executed
+ * @param count expected count
+ * @param keyword keyword to search
+ */
+ def checkExistenceCount(df: DataFrame, count: Long, keyword: String): Unit = {
+ val outputs = df.collect().map(_.mkString).mkString
+ assert(outputs.sliding(keyword.length).count(_ == keyword) === count)
+ }
+
def sqlTest(sqlString: String, expectedAnswer: Seq[Row])(implicit sqlContext: SQLContext) {
test(sqlString) {
checkAnswer(sqlContext.sql(sqlString), expectedAnswer)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b2b030e5/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index e78e49b..cddbd7a 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -620,6 +620,20 @@ class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll {
}
+ test("no inverted index after alter command") {
+ sql("drop table if exists NO_INVERTED_CARBON")
+ sql(
+ """
+ CREATE TABLE IF NOT EXISTS NO_INVERTED_CARBON
+ (id Int, name String, city String)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES('NO_INVERTED_INDEX'='city')
+ """)
+
+ sql("alter table NO_INVERTED_CARBON add columns(col1 string,col2 string) tblproperties('NO_INVERTED_INDEX'='col2')")
+ checkExistenceCount(sql("desc formatted NO_INVERTED_CARBON"),2,"NOINVERTEDINDEX")
+ }
+
test("test if adding column in pre-aggregate table throws exception") {
sql("drop table if exists preaggMain")
sql("drop table if exists preagg1")
@@ -647,6 +661,7 @@ class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS alter_dict")
sql("DROP TABLE IF EXISTS alter_sort_columns")
sql("DROP TABLE IF EXISTS alter_no_dict")
+ sql("drop table if exists NO_INVERTED_CARBON")
sqlContext.setConf("carbon.enable.vector.reader", "false")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)