You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/01/30 09:35:31 UTC
[carbondata] branch master updated: [CARBONDATA-3280] Fix the issue
of SDK assert can't work
This is an automated email from the ASF dual-hosted git repository.
kumarvishal09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new ba139b6 [CARBONDATA-3280] Fix the issue of SDK assert can't work
ba139b6 is described below
commit ba139b642d266e0767fedd4fb53c16d198b26d35
Author: xubo245 <xu...@huawei.com>
AuthorDate: Tue Jan 29 11:36:48 2019 +0800
[CARBONDATA-3280] Fix the issue of SDK assert can't work
After PR-3097 merged, the batch rule has been changed, but the test didn't work, such as:
org.apache.carbondata.sdk.file.CarbonReaderTest#testReadNextBatchRow
org.apache.carbondata.sdk.file.CarbonReaderTest#testReadNextBatchRowWithVectorReader
So this PR fixed the test error and add some assert
This closes #3112
---
.../carbondata/core/util/CarbonProperties.java | 2 +-
.../carbondata/sdk/file/CarbonReaderTest.java | 90 +++++++++++++++-------
2 files changed, 63 insertions(+), 29 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 49388b7..b337e40 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1572,7 +1572,7 @@ public final class CarbonProperties {
try {
batchSize = Integer.parseInt(batchSizeString);
if (batchSize < DETAIL_QUERY_BATCH_SIZE_MIN || batchSize > DETAIL_QUERY_BATCH_SIZE_MAX) {
- LOGGER.info("Invalid carbon.detail.batch.size.Using default value "
+ LOGGER.warn("Invalid carbon.detail.batch.size.Using default value "
+ DETAIL_QUERY_BATCH_SIZE_DEFAULT);
carbonProperties.setProperty(DETAIL_QUERY_BATCH_SIZE,
Integer.toString(DETAIL_QUERY_BATCH_SIZE_DEFAULT));
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 28944da..871d51b 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -104,7 +104,7 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File(path));
}
- @Test public void testReadWithZeroBatchSize() throws IOException, InterruptedException {
+ @Test public void testReadWithZeroBatchSize() throws Exception {
String path = "./testWriteFiles";
FileUtils.deleteDirectory(new File(path));
DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path));
@@ -127,6 +127,30 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File(path));
}
+
+ @Test
+ public void testReadBatchWithZeroBatchSize() throws Exception {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+ DataMapStoreManager.getInstance().clearDataMaps(AbsoluteTableIdentifier.from(path));
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ TestUtil.writeFilesAndVerify(10, new Schema(fields), path);
+ CarbonReader reader;
+ reader = CarbonReader.builder(path).withRowRecordReader().withBatch(0).build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] row = reader.readNextBatchRow();
+ Assert.assertEquals(row.length, 10);
+ i++;
+ }
+ Assert.assertEquals(i, 1);
+ FileUtils.deleteDirectory(new File(path));
+ }
+
@Test
public void testReadWithFilterOfNonTransactionalSimple() throws IOException, InterruptedException {
String path = "./testWriteFiles";
@@ -532,6 +556,7 @@ public class CarbonReaderTest extends TestCase {
.withCsvInput(schema).writtenBy("CarbonReaderTest").build();
} catch (InvalidLoadOptionException e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
}
carbonWriter.write(new String[] { "MNO", "100" });
carbonWriter.close();
@@ -546,22 +571,25 @@ public class CarbonReaderTest extends TestCase {
.withCsvInput(schema1).writtenBy("CarbonReaderTest").build();
} catch (InvalidLoadOptionException e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
}
carbonWriter1.write(new String[] { "PQR", "200" });
carbonWriter1.close();
try {
- CarbonReader reader =
- CarbonReader.builder(path1, "_temp").
- projection(new String[] { "c1", "c3" })
- .build();
- } catch (Exception e){
- System.out.println("Success");
+ CarbonReader reader =
+ CarbonReader.builder(path1, "_temp")
+ .projection(new String[]{"c1", "c3"})
+ .build();
+ Assert.fail();
+ } catch (Exception e) {
+ System.out.println("Success");
+ Assert.assertTrue(true);
}
CarbonReader reader1 =
- CarbonReader.builder(path2, "_temp1")
- .projection(new String[] { "p1", "p2" })
- .build();
+ CarbonReader.builder(path2, "_temp1")
+ .projection(new String[]{"p1", "p2"})
+ .build();
while (reader1.hasNext()) {
Object[] row1 = (Object[]) reader1.readNextRow();
@@ -1292,6 +1320,7 @@ public class CarbonReaderTest extends TestCase {
WriteAvroComplexData(mySchema, json, path);
} catch (InvalidLoadOptionException e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
}
File folder = new File(path);
@@ -1357,6 +1386,7 @@ public class CarbonReaderTest extends TestCase {
WriteAvroComplexData(mySchema, json, path);
} catch (InvalidLoadOptionException e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
}
Field[] fields = new Field[3];
@@ -1509,6 +1539,7 @@ public class CarbonReaderTest extends TestCase {
FileUtils.deleteDirectory(new File(path));
} catch (Throwable e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
}
}
@@ -1833,9 +1864,9 @@ public class CarbonReaderTest extends TestCase {
.writtenBy("CarbonReaderTest")
.build();
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 300; i++) {
String[] row2 = new String[]{
- "robot" + (i % 10),
+ "robot" + (i % 10000),
String.valueOf(i % 10000),
String.valueOf(i),
String.valueOf(Long.MAX_VALUE - i),
@@ -1852,11 +1883,11 @@ public class CarbonReaderTest extends TestCase {
}
writer.close();
- // Read data
- int batchSize =4;
+ // Read data
+ int batchSize = 150;
CarbonReader reader = CarbonReader
.builder(path, "_temp")
- .withBatch(4)
+ .withBatch(batchSize)
.build();
int i = 0;
@@ -1889,6 +1920,7 @@ public class CarbonReaderTest extends TestCase {
reader.close();
} catch (Throwable e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
} finally {
try {
FileUtils.deleteDirectory(new File(path));
@@ -1897,6 +1929,7 @@ public class CarbonReaderTest extends TestCase {
}
}
}
+
@Test
public void testReadNextBatchRowWithVectorReader() {
String path = "./carbondata";
@@ -1926,9 +1959,9 @@ public class CarbonReaderTest extends TestCase {
.writtenBy("CarbonReaderTest")
.build();
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 300; i++) {
String[] row2 = new String[]{
- "robot" + (i % 10),
+ "robot" + (i % 10000),
String.valueOf(i % 10000),
String.valueOf(i),
String.valueOf(Long.MAX_VALUE - i),
@@ -1945,10 +1978,10 @@ public class CarbonReaderTest extends TestCase {
writer.close();
// Read data
- int batchSize =4;
+ int batchSize = 150;
CarbonReader reader = CarbonReader
.builder(path, "_temp")
- .withBatch(4)
+ .withBatch(batchSize)
.build();
int i = 0;
@@ -1975,6 +2008,7 @@ public class CarbonReaderTest extends TestCase {
reader.close();
} catch (Throwable e) {
e.printStackTrace();
+ Assert.fail(e.getMessage());
} finally {
try {
FileUtils.deleteDirectory(new File(path));
@@ -2114,9 +2148,9 @@ public class CarbonReaderTest extends TestCase {
.build();
} catch (IllegalArgumentException e) {
e.printStackTrace();
- Assert.fail();
+ Assert.fail(e.getMessage());
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2144,7 +2178,7 @@ public class CarbonReaderTest extends TestCase {
Assert.assertTrue(e.getMessage().contains(
"Invalid value FLSE for key bad_records_logger_enable"));
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2171,7 +2205,7 @@ public class CarbonReaderTest extends TestCase {
e.printStackTrace();
Assert.fail();
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2199,7 +2233,7 @@ public class CarbonReaderTest extends TestCase {
Assert.assertTrue(e.getMessage().contains(
"QUOTECHAR cannot be more than one character."));
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2226,7 +2260,7 @@ public class CarbonReaderTest extends TestCase {
e.printStackTrace();
Assert.fail();
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2254,7 +2288,7 @@ public class CarbonReaderTest extends TestCase {
Assert.assertTrue(e.getMessage().contains(
"ESCAPECHAR cannot be more than one character."));
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}
@@ -2279,9 +2313,9 @@ public class CarbonReaderTest extends TestCase {
.build();
} catch (IllegalArgumentException e) {
e.printStackTrace();
- Assert.fail();
+ Assert.fail(e.getMessage());
} catch (Exception e) {
- Assert.fail();
+ Assert.fail(e.getMessage());
} finally {
FileUtils.deleteDirectory(new File(path));
}