You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2019/01/30 10:39:13 UTC
[carbondata] 09/27: [CARBONDATA-3262] Fix merge index failure
handling for compacted segment
This is an automated email from the ASF dual-hosted git repository.
ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit 8b872ebaa36dd92952cc6c61d413e823d2f62bc8
Author: kunal642 <ku...@gmail.com>
AuthorDate: Mon Jan 21 20:45:24 2019 +0530
[CARBONDATA-3262] Fix merge index failure handling for compacted segment
Problem: When merge index file writing fails, the load details for the segments being merged is wrongly written with the merged segment file name.
Due to this when the next load happens the segment file for merged segment is deleted.
Solution: Dont throw exception when merge index fails.
This closes #3090
---
.../core/writer/CarbonIndexFileMergeWriter.java | 47 +++++++++++++---------
.../spark/rdd/CarbonDataRDDFactory.scala | 9 +++--
.../command/management/CarbonLoadDataCommand.scala | 9 +++--
.../processing/loading/model/CarbonLoadModel.java | 6 +++
4 files changed, 45 insertions(+), 26 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
index 9dde6b7..80e0af5 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonIndexFileMergeWriter.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -39,6 +40,7 @@ import org.apache.carbondata.format.MergedBlockIndex;
import org.apache.carbondata.format.MergedBlockIndexHeader;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
public class CarbonIndexFileMergeWriter {
@@ -52,6 +54,8 @@ public class CarbonIndexFileMergeWriter {
*/
private ThriftWriter thriftWriter;
+ private Logger LOGGER = LogServiceFactory.getLogService(this.getClass().getCanonicalName());
+
public CarbonIndexFileMergeWriter(CarbonTable table) {
this.table = table;
}
@@ -68,27 +72,32 @@ public class CarbonIndexFileMergeWriter {
*/
private String mergeCarbonIndexFilesOfSegment(String segmentId,
String tablePath, List<String> indexFileNamesTobeAdded,
- boolean readFileFooterFromCarbonDataFile, String uuid) throws IOException {
- Segment segment = Segment.getSegment(segmentId, tablePath);
- String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
- CarbonFile[] indexFiles;
- SegmentFileStore sfs = null;
- if (segment != null && segment.getSegmentFileName() != null) {
- sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
- List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
- indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
- } else {
- indexFiles =
- SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, FileFactory.getConfiguration());
- }
- if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
- if (sfs == null) {
- return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded,
- readFileFooterFromCarbonDataFile, segmentPath, indexFiles, segmentId);
+ boolean readFileFooterFromCarbonDataFile, String uuid) {
+ try {
+ Segment segment = Segment.getSegment(segmentId, tablePath);
+ String segmentPath = CarbonTablePath.getSegmentPath(tablePath, segmentId);
+ CarbonFile[] indexFiles;
+ SegmentFileStore sfs = null;
+ if (segment != null && segment.getSegmentFileName() != null) {
+ sfs = new SegmentFileStore(tablePath, segment.getSegmentFileName());
+ List<CarbonFile> indexCarbonFiles = sfs.getIndexCarbonFiles();
+ indexFiles = indexCarbonFiles.toArray(new CarbonFile[indexCarbonFiles.size()]);
} else {
- return writeMergeIndexFileBasedOnSegmentFile(
- segmentId, indexFileNamesTobeAdded, sfs, indexFiles, uuid);
+ indexFiles =
+ SegmentIndexFileStore.getCarbonIndexFiles(segmentPath, FileFactory.getConfiguration());
+ }
+ if (isCarbonIndexFilePresent(indexFiles) || indexFileNamesTobeAdded != null) {
+ if (sfs == null) {
+ return writeMergeIndexFileBasedOnSegmentFolder(indexFileNamesTobeAdded,
+ readFileFooterFromCarbonDataFile, segmentPath, indexFiles, segmentId);
+ } else {
+ return writeMergeIndexFileBasedOnSegmentFile(segmentId, indexFileNamesTobeAdded, sfs,
+ indexFiles, uuid);
+ }
}
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to merge index files in path: " + tablePath, e);
}
return null;
}
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 02acde6..8268379 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -579,7 +579,8 @@ object CarbonDataRDDFactory {
}
val compactedSegments = new util.ArrayList[String]()
handleSegmentMerging(sqlContext,
- carbonLoadModel,
+ carbonLoadModel
+ .getCopyWithPartition(carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter),
carbonTable,
compactedSegments,
operationContext)
@@ -587,8 +588,10 @@ object CarbonDataRDDFactory {
writtenSegment
} catch {
case e: Exception =>
- throw new Exception(
- "Dataload is success. Auto-Compaction has failed. Please check logs.")
+ LOGGER.error(
+ "Auto-Compaction has failed. Ignoring this exception because the" +
+ " load is passed.", e)
+ writtenSegment
}
}
}
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 242a467..7e3ea90 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -874,16 +874,17 @@ case class CarbonLoadDataCommand(
// Trigger auto compaction
CarbonDataRDDFactory.handleSegmentMerging(
sparkSession.sqlContext,
- carbonLoadModel,
+ carbonLoadModel
+ .getCopyWithPartition(carbonLoadModel.getCsvHeader, carbonLoadModel.getCsvDelimiter),
table,
compactedSegments,
operationContext)
carbonLoadModel.setMergedSegmentIds(compactedSegments)
} catch {
case e: Exception =>
- throw new Exception(
- "Dataload is success. Auto-Compaction has failed. Please check logs.",
- e)
+ LOGGER.error(
+ "Auto-Compaction has failed. Ignoring this exception because the " +
+ "load is passed.", e)
}
val specs =
SegmentFileStore.getPartitionSpecs(carbonLoadModel.getSegmentId, carbonLoadModel.getTablePath)
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index ccf6eb2..1ce8aae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -488,6 +488,9 @@ public class CarbonLoadModel implements Serializable {
copy.parentTablePath = parentTablePath;
copy.sdkWriterCores = sdkWriterCores;
copy.columnCompressor = columnCompressor;
+ copy.rangePartitionColumn = rangePartitionColumn;
+ copy.scaleFactor = scaleFactor;
+ copy.totalSize = totalSize;
return copy;
}
@@ -544,6 +547,9 @@ public class CarbonLoadModel implements Serializable {
copyObj.parentTablePath = parentTablePath;
copyObj.sdkWriterCores = sdkWriterCores;
copyObj.columnCompressor = columnCompressor;
+ copyObj.rangePartitionColumn = rangePartitionColumn;
+ copyObj.scaleFactor = scaleFactor;
+ copyObj.totalSize = totalSize;
return copyObj;
}