You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/04/26 14:31:08 UTC
carbondata git commit: [CARBONDATA-2275]Query Failed for 0 byte
deletedelta file
Repository: carbondata
Updated Branches:
refs/heads/master 25d12d954 -> 184935827
[CARBONDATA-2275]Query Failed for 0 byte deletedelta file
When delete is failed on write step because of any exception from hdfs. Currently 0 bye deletedelta file is created and not getting cleaned up .
So when any Select Query is triggered , Select Query is failed
Root Cause:-
deletedelta of 0 size was considered for reader and while getting the hostname it has thrown IndexOutofBound.
Even Writer is failed while flush /close (exception from hdfs) but Exception is not thrown back to caller which is making delete operation is success.
Solution :-
While reading ignore 0 size deletedelta file When Writer is failed ,Throw Failure Exception to caller and show error on query client .
deletedelta of 0 byte will be handled in cleanup automatically in next delete/update .
This closes #2121
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/18493582
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/18493582
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/18493582
Branch: refs/heads/master
Commit: 1849358278df7e5c738e7497e2d0d2be26f87d5a
Parents: 25d12d9
Author: BJangir <ba...@gmail.com>
Authored: Fri Mar 23 22:49:27 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu Apr 26 20:03:49 2018 +0530
----------------------------------------------------------------------
.../datastore/filesystem/AbstractDFSCarbonFile.java | 5 ++++-
.../core/mutate/DeleteDeltaBlockDetails.java | 14 +++++++++-----
.../statusmanager/SegmentUpdateStatusManager.java | 5 +++--
.../core/writer/CarbonDeleteDeltaWriterImpl.java | 12 +++++++++---
4 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/18493582/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index 03e3de4..0419405 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -533,7 +533,10 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
RemoteIterator<LocatedFileStatus> iter =
path.getFileSystem(FileFactory.getConfiguration()).listLocatedStatus(path);
while (iter.hasNext()) {
- listStatus.add(iter.next());
+ LocatedFileStatus fileStatus = iter.next();
+ if (fileStatus.getLen() > 0) {
+ listStatus.add(fileStatus);
+ }
}
return getFiles(listStatus.toArray(new FileStatus[listStatus.size()]));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/18493582/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index 20db937..b0a0139 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import org.apache.carbondata.common.logging.LogService;
@@ -63,12 +64,15 @@ public class DeleteDeltaBlockDetails implements Serializable {
}
public List<DeleteDeltaBlockletDetails> getBlockletDetails() {
-
List<DeleteDeltaBlockletDetails> deleteDeltaBlockletDetailsList = new ArrayList<>();
- Iterator<Map.Entry<String, DeleteDeltaBlockletDetails>> iterator =
- blockletDetailsMap.entrySet().iterator();
- while (iterator.hasNext()) {
- deleteDeltaBlockletDetailsList.add(iterator.next().getValue());
+ if (null != blockletDetailsMap) {
+ Set<Map.Entry<String, DeleteDeltaBlockletDetails>> entries = blockletDetailsMap.entrySet();
+ if (null != entries && !entries.isEmpty()) {
+ Iterator<Map.Entry<String, DeleteDeltaBlockletDetails>> iterator = entries.iterator();
+ while (iterator.hasNext()) {
+ deleteDeltaBlockletDetailsList.add(iterator.next().getValue());
+ }
+ }
}
return deleteDeltaBlockletDetailsList;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/18493582/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 8c24bd1..363b5bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -352,7 +352,7 @@ public class SegmentUpdateStatusManager {
@Override public boolean accept(CarbonFile pathName) {
String fileName = pathName.getName();
- if (fileName.endsWith(extension)) {
+ if (fileName.endsWith(extension) && pathName.getSize() > 0) {
String firstPart = fileName.substring(0, fileName.indexOf('.'));
String blockName =
firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
@@ -402,7 +402,8 @@ public class SegmentUpdateStatusManager {
@Override public boolean accept(CarbonFile pathName) {
String fileName = pathName.getName();
- if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
+ if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)
+ && pathName.getSize() > 0) {
String firstPart = fileName.substring(0, fileName.indexOf('.'));
String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
long timestamp = Long.parseLong(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/18493582/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 1802515..506d290 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
-import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
@@ -74,6 +73,7 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
brWriter.write(value);
} catch (IOException ioe) {
LOGGER.error("Error message: " + ioe.getLocalizedMessage());
+ throw ioe;
} finally {
if (null != brWriter) {
brWriter.flush();
@@ -81,7 +81,10 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
if (null != dataOutStream) {
dataOutStream.flush();
}
- CarbonUtil.closeStreams(brWriter, dataOutStream);
+ if (null != brWriter) {
+ brWriter.close();
+ }
+
}
}
@@ -103,6 +106,7 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
brWriter.write(deletedData);
} catch (IOException ioe) {
LOGGER.error("Error message: " + ioe.getLocalizedMessage());
+ throw ioe;
} finally {
if (null != brWriter) {
brWriter.flush();
@@ -110,7 +114,9 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
if (null != dataOutStream) {
dataOutStream.flush();
}
- CarbonUtil.closeStreams(brWriter, dataOutStream);
+ if (null != brWriter) {
+ brWriter.close();
+ }
}
}