You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/10/24 02:21:51 UTC
carbondata git commit: [CARBONDATA-3030] Remove no use parameter in
test case
Repository: carbondata
Updated Branches:
refs/heads/master fa9a4eeeb -> 8af737204
[CARBONDATA-3030] Remove no use parameter in test case
Remove no use parameter in test case
This closes #2839
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8af73720
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8af73720
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8af73720
Branch: refs/heads/master
Commit: 8af7372042469f467d74c5b90047e401c0dc778e
Parents: fa9a4ee
Author: xubo245 <xu...@huawei.com>
Authored: Mon Oct 22 12:06:08 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Oct 24 10:21:24 2018 +0800
----------------------------------------------------------------------
docs/sdk-guide.md | 2 +-
.../carbondata/examples/S3UsingSDkExample.scala | 3 +-
.../TestNonTransactionalCarbonTable.scala | 37 ++++++++++----------
.../apache/carbondata/sdk/file/TestUtil.java | 15 +++-----
.../carbondata/sdk/file/CarbonReaderTest.java | 24 ++++++-------
5 files changed, 36 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/docs/sdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index be42b3f..3a9be71 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -28,7 +28,7 @@ In the carbon jars package, there exist a carbondata-store-sdk-x.x.x-SNAPSHOT.ja
This SDK writer, writes carbondata file and carbonindex file at a given path.
External client can make use of this writer to convert other format data or live data to create carbondata and index files.
-These SDK writer output contains just a carbondata and carbonindex files. No metadata folder will be present.
+These SDK writer output contains just carbondata and carbonindex files. No metadata folder will be present.
## Quick example
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
index 014bec5..666db1f 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/S3UsingSDkExample.scala
@@ -34,8 +34,7 @@ object S3UsingSDKExample {
// prepare SDK writer output
def buildTestData(
path: String,
- num: Int = 3,
- persistSchema: Boolean = false): Any = {
+ num: Int = 3): Any = {
// getCanonicalPath gives path with \, but the code expects /.
val writerPath = path.replace("\\", "/");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 0c55d1b..38cf201 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -68,46 +68,46 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
def buildTestDataSingleFile(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestData(3, false, null)
+ buildTestData(3, null)
}
def buildTestDataMultipleFiles(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestData(1000000, false, null)
+ buildTestData(1000000, null)
}
def buildTestDataTwice(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestData(3, false, null)
- buildTestData(3, false, null)
+ buildTestData(3, null)
+ buildTestData(3, null)
}
def buildTestDataSameDirectory(): Any = {
- buildTestData(3, false, null)
+ buildTestData(3, null)
}
def buildTestDataWithBadRecordForce(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
var options = Map("bAd_RECords_action" -> "FORCE").asJava
- buildTestData(3, false, options)
+ buildTestData(3, options)
}
def buildTestDataWithBadRecordFail(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
var options = Map("bAd_RECords_action" -> "FAIL").asJava
- buildTestData(15001, false, options)
+ buildTestData(15001, options)
}
def buildTestDataWithBadRecordIgnore(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
var options = Map("bAd_RECords_action" -> "IGNORE").asJava
- buildTestData(3, false, options)
+ buildTestData(3, options)
}
def buildTestDataWithBadRecordRedirect(): Any = {
FileUtils.deleteDirectory(new File(writerPath))
var options = Map("bAd_RECords_action" -> "REDIRECT").asJava
- buildTestData(3, false, options)
+ buildTestData(3, options)
}
def buildTestDataWithSortColumns(sortColumns: List[String]): Any = {
@@ -115,7 +115,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
buildTestData(3, null, sortColumns)
}
- def buildTestData(rows: Int, persistSchema: Boolean, options: util.Map[String, String]): Any = {
+ def buildTestData(rows: Int, options: util.Map[String, String]): Any = {
buildTestData(rows, options, List("name"))
}
@@ -194,7 +194,6 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// prepare sdk writer output
def buildTestDataWithSameUUID(rows: Int,
- persistSchema: Boolean,
options: util.Map[String, String],
sortColumns: List[String]): Any = {
val schema = new StringBuilder()
@@ -297,7 +296,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
"Read two sdk writer outputs before and after deleting the existing files and creating new " +
"files with same schema and UUID") {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestDataWithSameUUID(3, false, null, List("name"))
+ buildTestDataWithSameUUID(3, null, List("name"))
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -319,7 +318,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
// and creation of new file can happen at same timestamp.
Thread.sleep(1000)
assert(!new File(writerPath).exists())
- buildTestDataWithSameUUID(4, false, null, List("name"))
+ buildTestDataWithSameUUID(4, null, List("name"))
checkAnswer(sql("select * from sdkOutputTable"), Seq(
Row("robot0", 0, 0.0),
Row("robot1", 1, 0.5),
@@ -334,7 +333,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
test(" test csv fileheader for transactional table") {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestDataWithSameUUID(3, false, null, List("Name"))
+ buildTestDataWithSameUUID(3, null, List("Name"))
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -363,14 +362,14 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
test("test count star with multiple loads files with same schema and UUID") {
FileUtils.deleteDirectory(new File(writerPath))
- buildTestDataWithSameUUID(3, false, null, List("namE"))
+ buildTestDataWithSameUUID(3, null, List("namE"))
assert(new File(writerPath).exists())
sql("DROP TABLE IF EXISTS sdkOutputTable")
sql(
s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY 'carbondata' LOCATION
|'$writerPath' """.stripMargin)
checkAnswer(sql(s"""select count(*) from sdkOutputTable """), Seq(Row(3)))
- buildTestDataWithSameUUID(3, false, null, List("name"))
+ buildTestDataWithSameUUID(3, null, List("name"))
// should reflect new count
checkAnswer(sql(s"""select count(*) from sdkOutputTable """), Seq(Row(6)))
sql("DROP TABLE sdkOutputTable")
@@ -841,7 +840,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
Row("robot1", 1, 0.5),
Row("robot2", 2, 1.0)))
- buildTestDataWithSameUUID(3, false, null, List("name"))
+ buildTestDataWithSameUUID(3, null, List("name"))
checkAnswer(sql("select * from sdkOutputTable"), Seq(
Row("robot0", 0, 0.0),
@@ -854,7 +853,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
Row("robot1", 1, 0.5),
Row("robot2", 2, 1.0)))
- buildTestDataWithSameUUID(3, false, null, List("name"))
+ buildTestDataWithSameUUID(3, null, List("name"))
checkAnswer(sql("select * from sdkOutputTable"), Seq(
Row("robot0", 0, 0.0),
@@ -906,7 +905,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
Row("robot2", 2, 1.0)))
FileUtils.deleteDirectory(new File(writerPath))
- buildTestData(4, false, null)
+ buildTestData(4, null)
checkAnswer(sql("select * from sdkOutputTable"), Seq(
Row("robot0", 0, 0.0),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
index dc6e012..df31f9f 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/TestUtil.java
@@ -72,11 +72,6 @@ public class TestUtil {
100, schema, path, sortColumns, -1, -1);
}
- public static void writeFilesAndVerify(int rows, Schema schema, String path) {
- writeFilesAndVerify(
- rows, schema, path, null, -1, -1);
- }
-
public static void writeFilesAndVerify(Schema schema, String path) {
writeFilesAndVerify(
100, schema, path, null, -1, -1);
@@ -85,14 +80,12 @@ public class TestUtil {
/**
* write file and verify
*
- * @param rows number of rows
- * @param schema schema
- * @param path table store path
- * @param persistSchema whether persist schema
- * @param isTransactionalTable whether is transactional table
+ * @param rows number of rows
+ * @param schema schema
+ * @param path table store path
*/
public static void writeFilesAndVerify(
- int rows, Schema schema, String path, boolean persistSchema, boolean isTransactionalTable) {
+ int rows, Schema schema, String path) {
writeFilesAndVerify(rows, schema, path, null, -1, -1);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8af73720/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 58f3f61..f37b832 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -132,7 +132,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("name", DataTypes.STRING);
EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
@@ -167,7 +167,7 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("age", DataTypes.INT);
@@ -204,7 +204,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
@@ -246,7 +246,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
EqualToExpression equalToExpression = new EqualToExpression(columnExpression,
@@ -288,7 +288,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
GreaterThanExpression greaterThanExpression = new GreaterThanExpression(columnExpression,
@@ -330,7 +330,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
LessThanExpression lessThanExpression = new LessThanExpression(columnExpression,
@@ -372,7 +372,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
LessThanExpression lessThanExpression = new LessThanExpression(columnExpression,
@@ -414,7 +414,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
LessThanExpression lessThanExpression = new LessThanExpression(columnExpression,
@@ -456,7 +456,7 @@ public class CarbonReaderTest extends TestCase {
fields[1] = new Field("age", DataTypes.INT);
fields[2] = new Field("doubleField", DataTypes.DOUBLE);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
LessThanExpression lessThanExpression = new LessThanExpression(columnExpression,
@@ -1408,9 +1408,9 @@ public class CarbonReaderTest extends TestCase {
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path1, false, false);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path2, false, false);
- TestUtil.writeFilesAndVerify(200, new Schema(fields), path3, false, false);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path1);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path2);
+ TestUtil.writeFilesAndVerify(200, new Schema(fields), path3);
EqualToExpression equalToExpression = new EqualToExpression(
new ColumnExpression("name", DataTypes.STRING),