You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/10/31 08:27:19 UTC
carbondata git commit: [CARBONDATA-3051] Fix bugs in unclosed streams
Repository: carbondata
Updated Branches:
refs/heads/master 43ffe79f3 -> 0e39abf81
[CARBONDATA-3051] Fix bugs in unclosed streams
Failed to run CarbonReaderTests in windows env. This is due to two
problems in carbondata:
Problem 1: While encountering problems, we should close all the streams instead of
only close current one.
Problem 2: Current FileReader is not singleton nor the fileCache
variable is a class variable, so it can only provide cache function if
we pass the object among the methods. After reading version info, the
fileReader cannot be used by other methods, so we must close it.
This closes #2871
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0e39abf8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0e39abf8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0e39abf8
Branch: refs/heads/master
Commit: 0e39abf81b0c7279abbecb9a4a387e447054a29a
Parents: 43ffe79
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Oct 29 15:33:10 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Oct 31 16:27:00 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java | 3 ++-
.../java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java | 1 +
.../java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java | 1 +
3 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 9651a8f..cc3fd72 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.util.CarbonSessionInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
import org.apache.carbondata.hadoop.api.CarbonFileInputFormat;
@@ -169,7 +170,7 @@ public class CarbonReaderBuilder {
reader.initialize(split, attempt);
readers.add(reader);
} catch (Exception e) {
- reader.close();
+ CarbonUtil.closeStreams(readers.toArray(new RecordReader[0]));
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
index 53ebc53..fe8e168 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java
@@ -93,6 +93,7 @@ public class CarbonSchemaReader {
FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath));
ByteBuffer buffer =
fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(dataFilePath), fileSize - 8, 8);
+ fileReader.finish();
CarbonFooterReaderV3 footerReader = new CarbonFooterReaderV3(dataFilePath, buffer.getLong());
FileFooter3 footer = footerReader.readFooterVersion3();
if (null != footer.getExtra_info()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e39abf8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index 90ea909..ed697d7 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -355,6 +355,7 @@ public class CSVCarbonWriterTest {
FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFile.getPath()));
ByteBuffer buffer = fileReader.readByteBuffer(FileFactory.getUpdatedFilePath(
dataFile.getPath()), dataFile.getSize() - 8, 8);
+ fileReader.finish();
CarbonFooterReaderV3 footerReader =
new CarbonFooterReaderV3(dataFile.getAbsolutePath(), buffer.getLong());
FileFooter3 footer = footerReader.readFooterVersion3();