You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/02/28 14:18:20 UTC
carbondata git commit: [CARBONDATA-2216][Test] Fix bugs in sdv tests
Repository: carbondata
Updated Branches:
refs/heads/datamap 47ae31bce -> 69eb26e68
[CARBONDATA-2216][Test] Fix bugs in sdv tests
This closes #2012
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/69eb26e6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/69eb26e6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/69eb26e6
Branch: refs/heads/datamap
Commit: 69eb26e68ca3252cca72a802f88cd3adfc01165d
Parents: 47ae31b
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Feb 28 16:02:55 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Feb 28 22:18:08 2018 +0800
----------------------------------------------------------------------
.../sdv/generated/MergeIndexTestCase.scala | 33 ++++++++++++++++----
1 file changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/69eb26e6/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index 86fd240..e768e9b 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -18,6 +18,16 @@
package org.apache.carbondata.cluster.sdv.generated
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
/**
* Test Class for AlterTableTestCase to verify all scenerios
*/
@@ -84,25 +94,36 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
val rows = sql("""Select count(*) from carbon_automation_nonmerge""").collect()
- assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0") == 2)
- assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 2)
- assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 2)
+ assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0") == 1)
+ assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 1)
+ assert(getIndexFileCount("default", "carbon_automation_nonmerge", "2") == 1)
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, "true")
- sql("ALTER TABLE carbon_automation_nonmerge COMPACT 'minor'").collect()
+ sql("ALTER TABLE carbon_automation_nonmerge COMPACT 'major'").collect()
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 0)
+ assert(getMergedIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 1)
checkAnswer(sql("""Select count(*) from carbon_automation_nonmerge"""), rows)
}
private def getIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
+ getFileCount(dbName, tableName, segment, CarbonTablePath.INDEX_FILE_EXT)
+ }
+
+ private def getMergedIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
+ getFileCount(dbName, tableName, segment, CarbonTablePath.MERGE_INDEX_FILE_EXT)
+ }
+
+ private def getFileCount(dbName: String,
+ tableName: String,
+ segment: String,
+ suffix: String): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName, tableName)
val identifier = carbonTable.getAbsoluteTableIdentifier
val path = CarbonTablePath
.getSegmentPath(identifier.getTablePath, segment)
val carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = {
- file.getName.endsWith(CarbonTablePath
- .INDEX_FILE_EXT)
+ file.getName.endsWith(suffix)
}
})
if (carbonFiles != null) {