You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/29 06:41:42 UTC
carbondata git commit: [CARBONDATA-1947]fix select * issue after
compaction, delete and clean files operation
Repository: carbondata
Updated Branches:
refs/heads/master 1937a21a2 -> 83e92715a
[CARBONDATA-1947]fix select * issue after compaction, delete and clean files operation
When updating the metadata details for merged segment, get the load starttime from carbonloadModel facttime stamp
This closes #1735
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83e92715
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83e92715
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83e92715
Branch: refs/heads/master
Commit: 83e92715a6819d2ff5c3e5cd50fe073be5ee01a6
Parents: 1937a21
Author: akashrn5 <ak...@gmail.com>
Authored: Thu Dec 28 13:43:15 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Dec 29 14:41:30 2017 +0800
----------------------------------------------------------------------
.../testsuite/iud/DeleteCarbonTableTestCase.scala | 15 +++++++++++++++
.../carbondata/spark/rdd/CarbonTableCompactor.scala | 3 +--
.../processing/merger/CarbonDataMergerUtil.java | 4 ++--
3 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index deaad20..6521657 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -165,6 +165,21 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists preaggmain_preagg1")
}
+ test("test select query after compaction, delete and clean files") {
+ sql("drop table if exists select_after_clean")
+ sql("create table select_after_clean(id int, name string) stored by 'carbondata'")
+ sql("insert into select_after_clean select 1,'abc'")
+ sql("insert into select_after_clean select 2,'def'")
+ sql("insert into select_after_clean select 3,'uhj'")
+ sql("insert into select_after_clean select 4,'frg'")
+ sql("alter table select_after_clean compact 'minor'")
+ sql("clean files for table select_after_clean")
+ sql("delete from select_after_clean where name='def'")
+ sql("clean files for table select_after_clean")
+ checkAnswer(sql("""select * from select_after_clean"""),
+ Seq(Row(1, "abc"), Row(3, "uhj"), Row(4, "frg")))
+ }
+
override def afterAll {
sql("use default")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 9768625..0dc856d 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -135,7 +135,6 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
val factTableName = carbonLoadModel.getTableName
val validSegments: Array[String] = CarbonDataMergerUtil
.getValidSegments(loadsToMerge).split(',')
- val mergeLoadStartTime = CarbonUpdateUtil.readCurrentTime()
val partitionMapper = if (carbonTable.isHivePartitionTable) {
var partitionMap: util.Map[String, util.List[String]] = null
validSegments.foreach { segmentId =>
@@ -245,7 +244,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
carbonLoadModel)) ||
CarbonDataMergerUtil
.updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath,
- mergedLoadNumber, carbonLoadModel, mergeLoadStartTime, compactionType)
+ mergedLoadNumber, carbonLoadModel, compactionType)
if (!statusFileUpdation) {
LOGGER.audit(s"Compaction request failed for table ${ carbonLoadModel.getDatabaseName }." +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index e245927..26648d0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -280,7 +280,7 @@ public final class CarbonDataMergerUtil {
*/
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge,
String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel,
- long mergeLoadStartTime, CompactionType compactionType) throws IOException {
+ CompactionType compactionType) throws IOException {
boolean tableStatusUpdationStatus = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -327,7 +327,7 @@ public final class CarbonDataMergerUtil {
loadMetadataDetails.setLoadName(mergedLoadNumber);
CarbonLoaderUtil
.addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable);
- loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
+ loadMetadataDetails.setLoadStartTime(carbonLoadModel.getFactTimeStamp());
loadMetadataDetails.setPartitionCount("0");
// if this is a major compaction then set the segment as major compaction.
if (CompactionType.MAJOR == compactionType) {