You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2019/03/27 10:29:44 UTC
[carbondata] branch master updated: [CARBONDATA-3330] Fix Invalid
Exception while clearing datamap from SDK carbon reader
This is an automated email from the ASF dual-hosted git repository.
ravipesala pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 35e63b1 [CARBONDATA-3330] Fix Invalid Exception while clearing datamap from SDK carbon reader
35e63b1 is described below
commit 35e63b1d5e308aa7a119278c13a6abfc353a172a
Author: ajantha-bhat <aj...@gmail.com>
AuthorDate: Mon Mar 25 16:27:14 2019 +0800
[CARBONDATA-3330] Fix Invalid Exception while clearing datamap from SDK carbon reader
problem:
When clearing the datamap in SDK carbon reader close(), it was always checking for schema file to build table. for SDK it will not find schema file and hence the exception.
cause : This was already an existing bug, now visible after #2878
Carbon table is required only for launching datamap clearing job, for SDK no need to launch the job
solution: Look for carbon table only if need to launch the datamap clearing job.
This closes #3162
---
.../core/datamap/DataMapStoreManager.java | 16 +++++----
.../carbondata/sdk/file/CarbonReaderTest.java | 38 +++++++++++-----------
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 524d8b0..a797b11 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -482,7 +482,6 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier, boolean launchJob) {
- CarbonTable carbonTable = getCarbonTable(identifier);
String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName);
if (tableIndices == null) {
@@ -492,12 +491,15 @@ public final class DataMapStoreManager {
tableIndices = allDataMaps.get(tableUniqueName);
}
}
- if (null != carbonTable && tableIndices != null && launchJob) {
- try {
- DataMapUtil.executeDataMapJobForClearingDataMaps(carbonTable);
- } catch (IOException e) {
- LOGGER.error("clear dataMap job failed", e);
- // ignoring the exception
+ if (launchJob && tableIndices != null) {
+ CarbonTable carbonTable = getCarbonTable(identifier);
+ if (null != carbonTable) {
+ try {
+ DataMapUtil.executeDataMapJobForClearingDataMaps(carbonTable);
+ } catch (IOException e) {
+ LOGGER.error("clear dataMap job failed", e);
+ // ignoring the exception
+ }
}
}
segmentRefreshMap.remove(identifier.uniqueName());
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 871d51b..f09581a 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -65,7 +65,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -107,7 +107,7 @@ public class CarbonReaderTest extends TestCase {
@Test public void testReadWithZeroBatchSize() throws Exception {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
- DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path));
+ DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -132,7 +132,7 @@ public class CarbonReaderTest extends TestCase {
public void testReadBatchWithZeroBatchSize() throws Exception {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
- DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path));
+ DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -156,7 +156,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
String path1 = path + "/0testdir";
String path2 = path + "/testdir";
@@ -203,7 +203,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -240,7 +240,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -283,7 +283,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -326,7 +326,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -369,7 +369,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -412,7 +412,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -462,7 +462,7 @@ public class CarbonReaderTest extends TestCase {
TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
LessThanExpression lessThanExpression = new LessThanExpression(columnExpression,
new LiteralExpression("13.5", DataTypes.DOUBLE));
@@ -498,7 +498,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[3];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -543,9 +543,9 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File(path1));
FileUtils.deleteDirectory(new File(path2));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path1));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path1), false);
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path2));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path2), false);
Field[] fields = new Field[] { new Field("c1", "string"),
new Field("c2", "int") };
Schema schema = new Schema(fields);
@@ -607,7 +607,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -644,7 +644,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -680,7 +680,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -721,7 +721,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -751,7 +751,7 @@ public class CarbonReaderTest extends TestCase {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance()
- .clearDataMaps(AbsoluteTableIdentifier.from(path));
+ .clearDataMaps(AbsoluteTableIdentifier.from(path), false);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);