You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:41:01 UTC
[36/50] [abbrv] carbondata git commit: [CARBONDATA-2555] SDK reader
set default isTransactional as false
[CARBONDATA-2555] SDK reader set default isTransactional as false
SDK writer is having default value of isTransactional is false. But reader is not like this.
So, Fixing this by deafult make SDK to use flat folder structure.
This closes #2352
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/74770aa3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/74770aa3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/74770aa3
Branch: refs/heads/spark-2.3
Commit: 74770aa38173985ece5527847f8e2446966e4fd7
Parents: a7faef8
Author: ajantha-bhat <aj...@gmail.com>
Authored: Tue May 29 12:03:55 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Wed May 30 10:26:19 2018 +0530
----------------------------------------------------------------------
docs/sdk-guide.md | 2 +-
.../examples/sdk/CarbonReaderExample.java | 1 -
.../carbondata/examples/sdk/SDKS3Example.java | 26 +++-----------------
.../sdk/file/CarbonReaderBuilder.java | 4 +--
.../carbondata/sdk/file/CarbonReaderTest.java | 20 +++++++++++----
5 files changed, 21 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 3c575fe..1d225a9 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -492,7 +492,7 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/
* Configure the transactional status of table
* If set to false, then reads the carbondata and carbonindex files from a flat folder structure.
* If set to true, then reads the carbondata and carbonindex files from segment folder structure.
- * Default value is true
+ * Default value is false
*
* @param isTransactionalTable whether is transactional table or not
* @return CarbonReaderBuilder object
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index 937bfa0..d7886c0 100644
--- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -44,7 +44,6 @@ public class CarbonReaderExample {
fields[1] = new Field("age", DataTypes.INT);
CarbonWriter writer = CarbonWriter.builder()
- .isTransactionalTable(true)
.outputPath(path)
.persistSchemaFile(true)
.buildWriterForCSVInput(new Schema(fields));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java
index 7fab2cc..80c56fc 100644
--- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java
+++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java
@@ -29,8 +29,8 @@ public class SDKS3Example {
public static void main(String[] args) throws Exception {
LogService logger = LogServiceFactory.getLogService(SDKS3Example.class.getName());
if (args == null || args.length < 3) {
- logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>" +
- "<s3-endpoint> [table-path-on-s3] [persistSchema] [transactionalTable]");
+ logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>"
+ + "<s3-endpoint> [table-path-on-s3] [rows]");
System.exit(0);
}
@@ -44,24 +44,6 @@ public class SDKS3Example {
num = Integer.parseInt(args[4]);
}
- Boolean persistSchema = true;
- if (args.length > 5) {
- if (args[5].equalsIgnoreCase("true")) {
- persistSchema = true;
- } else {
- persistSchema = false;
- }
- }
-
- Boolean transactionalTable = true;
- if (args.length > 6) {
- if (args[6].equalsIgnoreCase("true")) {
- transactionalTable = true;
- } else {
- transactionalTable = false;
- }
- }
-
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
@@ -69,9 +51,7 @@ public class SDKS3Example {
.setAccessKey(args[0])
.setSecretKey(args[1])
.setEndPoint(args[2])
- .outputPath(path)
- .persistSchemaFile(persistSchema)
- .isTransactionalTable(transactionalTable);
+ .outputPath(path);
CarbonWriter writer = builder.buildWriterForCSVInput(new Schema(fields));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 1e73e8c..e99ff0d 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -48,7 +48,7 @@ public class CarbonReaderBuilder {
private String[] projectionColumns;
private Expression filterExpression;
private String tableName;
- private boolean isTransactionalTable = true;
+ private boolean isTransactionalTable;
/**
* It will be true if use the projectAllColumns method，
@@ -84,7 +84,7 @@ public class CarbonReaderBuilder {
* Configure the transactional status of table
* If set to false, then reads the carbondata and carbonindex files from a flat folder structure.
* If set to true, then reads the carbondata and carbonindex files from segment folder structure.
- * Default value is true
+ * Default value is false
*
* @param isTransactionalTable whether is transactional table or not
* @return CarbonReaderBuilder object
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74770aa3/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index ee095a1..deb6d06 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -61,7 +61,7 @@ public class CarbonReaderTest extends TestCase {
TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
- CarbonReader reader = CarbonReader.builder(path, "_temp")
+ CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
.projection(new String[]{"name", "age"}).build();
// expected output after sorting
@@ -87,6 +87,7 @@ public class CarbonReaderTest extends TestCase {
// Read again
CarbonReader reader2 = CarbonReader
.builder(path, "_temp")
+ .isTransactionalTable(true)
.projection(new String[]{"name", "age"})
.build();
@@ -118,6 +119,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.projection(new String[]{"name", "name", "age", "name"})
+ .isTransactionalTable(true)
.build();
// expected output after sorting
@@ -159,11 +161,13 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.projection(new String[]{"name", "age"})
+ .isTransactionalTable(true)
.build();
// Reader 2
CarbonReader reader2 = CarbonReader
.builder(path, "_temp")
.projection(new String[]{"name", "age"})
+ .isTransactionalTable(true)
.build();
while (reader.hasNext()) {
@@ -191,7 +195,7 @@ public class CarbonReaderTest extends TestCase {
TestUtil.writeFilesAndVerify(new Schema(fields), path, true);
- CarbonReader reader = CarbonReader.builder(path, "_temp")
+ CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
.projection(new String[]{"name", "age"}).build();
reader.close();
@@ -305,7 +309,7 @@ public class CarbonReaderTest extends TestCase {
// Write to a Non Transactional Table
TestUtil.writeFilesAndVerify(new Schema(fields), path, true, false);
- CarbonReader reader = CarbonReader.builder(path, "_temp")
+ CarbonReader reader = CarbonReader.builder(path, "_temp").isTransactionalTable(true)
.projection(new String[]{"name", "age"})
.isTransactionalTable(false)
.build();
@@ -422,8 +426,8 @@ public class CarbonReaderTest extends TestCase {
Assert.assertNotNull(dataFiles);
Assert.assertTrue(dataFiles.length > 0);
- CarbonReader reader = CarbonReader
- .builder(path, "_temp")
+ CarbonReader reader = CarbonReader.builder(path, "_temp")
+ .isTransactionalTable(true)
.projection(new String[]{
"stringField"
, "shortField"
@@ -548,6 +552,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
+ .isTransactionalTable(true)
.projection(strings)
.build();
@@ -662,6 +667,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.projection(strings)
+ .isTransactionalTable(true)
.build();
int i = 0;
@@ -766,6 +772,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
+ .isTransactionalTable(true)
.projection(strings)
.build();
@@ -808,6 +815,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
+ .isTransactionalTable(true)
.projectAllColumns()
.build();
@@ -846,6 +854,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
+ .isTransactionalTable(true)
.build();
// expected output after sorting
@@ -881,6 +890,7 @@ public class CarbonReaderTest extends TestCase {
CarbonReader reader = CarbonReader
.builder(path, "_temp")
.projection(new String[]{})
+ .isTransactionalTable(true)
.build();
assert (false);
} catch (RuntimeException e) {