You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@carbondata.apache.org by GitBox <gi...@apache.org> on 2021/10/18 09:33:37 UTC
[GitHub] [carbondata] ShreelekhyaG commented on a change in pull request #4228: [CARBONDATA-4298][CARBONDATA-4281] Empty bad record support for complex type
ShreelekhyaG commented on a change in pull request #4228:
URL: https://github.com/apache/carbondata/pull/4228#discussion_r730741761
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordEmptyDataTest.scala
##########
@@ -101,6 +101,47 @@ class BadRecordEmptyDataTest extends QueryTest with BeforeAndAfterAll {
}
}
+ test("Test complex type with empty values and IS_EMPTY_DATA_BAD_RECORD property") {
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS complexhivetable")
+ sql(
+ "create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize " +
+ "string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:int>, MAC " +
+ "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, " +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) STORED AS carbondata")
+ sql("LOAD DATA local inpath '" + resourcesPath +
+ "/complextypeWithEmptyRecords.csv' INTO table complexcarbontable OPTIONS('DELIMITER'=','," +
+ "'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName," +
+ "purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', " +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':', " +
+ "'bad_records_logger_enable'='true','IS_EMPTY_DATA_BAD_RECORD'='true' ," +
+ "'bad_records_action'='ignore')")
Review comment:
Done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordEmptyDataTest.scala
##########
@@ -101,6 +101,47 @@ class BadRecordEmptyDataTest extends QueryTest with BeforeAndAfterAll {
}
}
+ test("Test complex type with empty values and IS_EMPTY_DATA_BAD_RECORD property") {
+ sql("DROP TABLE IF EXISTS complexcarbontable")
+ sql("DROP TABLE IF EXISTS complexhivetable")
+ sql(
+ "create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize " +
+ "string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:int>, MAC " +
+ "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@carbondata.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org