You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2016/08/31 16:05:44 UTC
[1/2] incubator-carbondata git commit: once the segment is compacted
then if user selects the compacted segment to delete then no need to delete
the segment. displaying error msg on this.
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 3eec687fe -> 1a0d2a672
once the segment is compacted then if user selects the compacted segment to delete then no need to delete the segment.
displaying error msg on this.
adding segment id in the error log.
handling in the delete segment by date.
removed duplicate code.
adding test cases for deletion of compacted segments scenario.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6580f4ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6580f4ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6580f4ad
Branch: refs/heads/master
Commit: 6580f4ad0a0ae521ce56f28b059eddedf97136f1
Parents: 3eec687
Author: ravikiran <ra...@gmail.com>
Authored: Wed Aug 24 19:36:06 2016 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Aug 31 21:32:33 2016 +0530
----------------------------------------------------------------------
.../MajorCompactionIgnoreInMinorTest.scala | 62 ++++++++++++++++++++
.../lcm/status/SegmentStatusManager.java | 16 ++++-
2 files changed, 76 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6580f4ad/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 867b7e4..a062153 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -20,12 +20,15 @@ package org.apache.carbondata.spark.testsuite.datacompaction
import java.io.File
+import org.apache.carbondata.core.carbon.path.CarbonStorePath
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.CarbonHiveContext._
import org.apache.spark.sql.common.util.QueryTest
import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.lcm.status
import org.apache.carbondata.lcm.status.SegmentStatusManager
import org.scalatest.BeforeAndAfterAll
@@ -140,6 +143,65 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
}
+ /**
+ * Delete should not work on compacted segment.
+ */
+ test("delete compacted segment and check status") {
+ try {
+ sql("delete segment 2 from table ignoremajor")
+ assert(false)
+ }
+ catch {
+ case _ => assert(true)
+ }
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", "rrr")
+ )
+ )
+ val carbontablePath = CarbonStorePath
+ .getCarbonTablePath(CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", "rrr")
+ )
+ .getMetadataDirectoryPath
+ var segs = segmentStatusManager.readLoadMetadata(carbontablePath)
+
+ // status should remain as compacted.
+ assert(segs(3).getLoadStatus.equalsIgnoreCase(CarbonCommonConstants.SEGMENT_COMPACTED))
+
+ }
+
+ /**
+ * Delete should not work on compacted segment.
+ */
+ test("delete compacted segment by date and check status") {
+ sql(
+ "DELETE SEGMENTS FROM TABLE ignoremajor where STARTTIME before" +
+ " '2222-01-01 19:35:01'"
+ )
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", "rrr")
+ )
+ )
+ val carbontablePath = CarbonStorePath
+ .getCarbonTablePath(CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", "rrr")
+ )
+ .getMetadataDirectoryPath
+ var segs = segmentStatusManager.readLoadMetadata(carbontablePath)
+
+ // status should remain as compacted for segment 2.
+ assert(segs(3).getLoadStatus.equalsIgnoreCase(CarbonCommonConstants.SEGMENT_COMPACTED))
+ // for segment 0.1 . should get deleted
+ assert(segs(2).getLoadStatus.equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE))
+
+ }
+
override def afterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6580f4ad/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
index 4b57de2..2b83b11 100644
--- a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
+++ b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java
@@ -410,18 +410,24 @@ public class SegmentStatusManager {
for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
if (loadId.equalsIgnoreCase(loadMetadata.getLoadName())) {
+ // if the segment is compacted then no need to delete that.
+ if (CarbonCommonConstants.SEGMENT_COMPACTED
+ .equalsIgnoreCase(loadMetadata.getLoadStatus())) {
+ LOG.error("Cannot delete the Segment which is compacted. Segment is " + loadId);
+ break;
+ }
if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())) {
loadFound = true;
loadMetadata.setLoadStatus(CarbonCommonConstants.MARKED_FOR_DELETE);
loadMetadata.setModificationOrdeletionTimesStamp(readCurrentTime());
- LOG.info("LoadId " + loadId + " Marked for Delete");
+ LOG.info("Segment ID " + loadId + " Marked for Delete");
}
break;
}
}
if (!loadFound) {
- LOG.audit("Delete load by Id is failed. No matching load id found.");
+ LOG.audit("Delete segment by ID is failed. No matching segment id found.");
invalidLoadIds.add(loadId);
return invalidLoadIds;
}
@@ -449,6 +455,12 @@ public class SegmentStatusManager {
for (LoadMetadataDetails loadMetadata : listOfLoadFolderDetailsArray) {
Integer result = compareDateValues(loadMetadata.getLoadStartTimeAsLong(), loadStartTime);
if (result < 0) {
+ if (CarbonCommonConstants.SEGMENT_COMPACTED
+ .equalsIgnoreCase(loadMetadata.getLoadStatus())) {
+ LOG.info("Ignoring the segment : " + loadMetadata.getLoadName()
+ + "as the segment has been compacted.");
+ continue;
+ }
if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())) {
loadFound = true;
loadMetadata.setLoadStatus(CarbonCommonConstants.MARKED_FOR_DELETE);
[2/2] incubator-carbondata git commit: [CARBONDATA-176] This closes
#92
Posted by gv...@apache.org.
[CARBONDATA-176] This closes #92
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1a0d2a67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1a0d2a67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1a0d2a67
Branch: refs/heads/master
Commit: 1a0d2a672acb2be18daff490d0c943dda9416bda
Parents: 3eec687 6580f4a
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Wed Aug 31 21:34:35 2016 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Aug 31 21:34:35 2016 +0530
----------------------------------------------------------------------
.../MajorCompactionIgnoreInMinorTest.scala | 62 ++++++++++++++++++++
.../lcm/status/SegmentStatusManager.java | 16 ++++-
2 files changed, 76 insertions(+), 2 deletions(-)
----------------------------------------------------------------------