You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/03/23 09:39:32 UTC
[1/2] incubator-carbondata git commit: added support to rename
badrecords while renaming table
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 67acca20e -> 10f6bf832
added support to rename badrecords while renaming table
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2f148867
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2f148867
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2f148867
Branch: refs/heads/master
Commit: 2f148867dfa35153888e7ccb8b4920be5a10558c
Parents: 67acca2
Author: kunal642 <ku...@knoldus.in>
Authored: Tue Mar 21 13:06:44 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Mar 23 15:02:25 2017 +0530
----------------------------------------------------------------------
.../execution/command/AlterTableCommands.scala | 22 +++++++++++++-
.../AlterTableValidationTestCase.scala | 32 ++++++++++++++++++++
2 files changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2f148867/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
index ee38cb9..efa2cd5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
@@ -22,7 +22,6 @@ import scala.collection.mutable.ListBuffer
import scala.language.implicitConversions
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hive.{CarbonRelation, HiveExternalCatalog}
import org.apache.spark.util.AlterTableUtil
@@ -35,6 +34,7 @@ import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifie
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.format.{ColumnSchema, SchemaEvolutionEntry, TableInfo}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -156,6 +156,7 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
.readSchemaFile(tableMetadataFile)
val schemaEvolutionEntry = new SchemaEvolutionEntry(System.currentTimeMillis)
schemaEvolutionEntry.setTableName(newTableName)
+ renameBadRecords(oldTableName, newTableName, oldDatabaseName)
val fileType = FileFactory.getFileType(tableMetadataFile)
if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
val rename = FileFactory.getCarbonFile(carbonTablePath.getPath, fileType)
@@ -163,6 +164,7 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
newTableName)
if (!rename) {
sys.error(s"Folder rename failed for table $oldDatabaseName.$oldTableName")
+ renameBadRecords(newTableName, oldTableName, oldDatabaseName)
}
}
val newTableIdentifier = new CarbonTableIdentifier(oldDatabaseName,
@@ -198,6 +200,24 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
}
Seq.empty
}
+
+ private def renameBadRecords(oldTableName: String,
+ newTableName: String,
+ dataBaseName: String) = {
+ val oldPath = CarbonUtil
+ .getBadLogPath(dataBaseName + CarbonCommonConstants.FILE_SEPARATOR + oldTableName)
+ val newPath = CarbonUtil
+ .getBadLogPath(dataBaseName + CarbonCommonConstants.FILE_SEPARATOR + newTableName)
+ val fileType = FileFactory.getFileType(oldPath)
+ if (FileFactory.isFileExist(oldPath, fileType)) {
+ val renameSuccess = FileFactory.getCarbonFile(oldPath, fileType)
+ .renameForce(newPath)
+ if (!renameSuccess) {
+ sys.error(s"BadRecords Folder Rename Failed for table $dataBaseName.$oldTableName")
+ }
+ }
+ }
+
}
private[sql] case class AlterTableDropColumns(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2f148867/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index 8f8f8cb..0a59497 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -1,18 +1,28 @@
package org.apache.spark.carbondata.restructure
+import java.io.File
import java.math.{BigDecimal, RoundingMode}
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ new File("./target/test/badRecords").getCanonicalPath)
+
sql("drop table if exists restructure")
+ sql("drop table if exists restructure_test")
+ sql("drop table if exists restructure_new")
+ sql("drop table if exists restructure_bad")
+ sql("drop table if exists restructure_badnew")
// clean data folder
CarbonProperties.getInstance()
sql(
@@ -31,6 +41,18 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql(
s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE restructure_test OPTIONS
|('DELIMITER'= ',', 'QUOTECHAR'= '\"')""".stripMargin)
+
+ sql(
+ """CREATE TABLE IF NOT EXISTS restructure_bad(ID BigInt, date Timestamp, country String,
+ actual_price Double, Quantity int, sold_price Decimal(19,2)) STORED BY 'carbondata'""")
+
+ sql(
+ s"""LOAD DATA LOCAL INPATH '$resourcesPath/badrecords/datasample.csv' INTO TABLE
+ |restructure_bad OPTIONS
+ |('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'bad_records_logger_enable'='true',
+ |'bad_records_action'='redirect')"""
+ .stripMargin)
+
}
test("test add dictionary column") {
@@ -302,6 +324,14 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
assert(result.count().equals(10L))
}
+ test("test to check if bad record folder name is changed") {
+ sql("alter table restructure_bad rename to restructure_badnew")
+ val oldLocation = new File("./target/test/badRecords/default/restructure_bad")
+ val newLocation = new File("./target/test/badRecords/default/restructure_badnew")
+ assert(!oldLocation.exists())
+ assert(newLocation.exists())
+ }
+
test("test to rename table with invalid table name") {
try {
sql("alter table restructure_invalid rename to restructure_new")
@@ -347,5 +377,7 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS restructure")
sql("DROP TABLE IF EXISTS restructure_new")
sql("DROP TABLE IF EXISTS restructure_test")
+ sql("DROP TABLE IF EXISTS restructure_bad")
+ sql("DROP TABLE IF EXISTS restructure_badnew")
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-798] Rename table
should rename bad records folder. This closes #678
Posted by gv...@apache.org.
[CARBONDATA-798] Rename table should rename bad records folder. This closes #678
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/10f6bf83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/10f6bf83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/10f6bf83
Branch: refs/heads/master
Commit: 10f6bf832d61fea0d66e0f9a1ed542a011d0504d
Parents: 67acca2 2f14886
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Thu Mar 23 15:09:17 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Mar 23 15:09:17 2017 +0530
----------------------------------------------------------------------
.../execution/command/AlterTableCommands.scala | 22 +++++++++++++-
.../AlterTableValidationTestCase.scala | 32 ++++++++++++++++++++
2 files changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------