You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/01/16 11:27:31 UTC
[1/2] incubator-carbondata git commit: support for float datatype
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 396d0d6b0 -> b5c20a80a
support for float datatype
remove style checks for float support
fix ordering imports for style checks
add testcases and example for float datatype
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/54e83045
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/54e83045
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/54e83045
Branch: refs/heads/master
Commit: 54e83045ee29ecbb67ab08eaaab480fa6e7b09a4
Parents: 396d0d6
Author: sandy <ph...@gmail.com>
Authored: Wed Jan 11 15:07:38 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Jan 16 16:56:00 2017 +0530
----------------------------------------------------------------------
examples/spark/src/main/resources/data.csv | 22 ++++----
.../examples/AllDictionaryExample.scala | 6 ++-
examples/spark2/src/main/resources/data.csv | 20 +++----
.../examples/CarbonSessionExample.scala | 11 +++-
.../src/test/resources/floatSample.csv | 11 ++++
.../primitiveTypes/FloatDataTypeTestCase.scala | 50 +++++++++++++++++
.../spark/util/DataTypeConverterUtil.scala | 3 ++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 9 +++-
.../spark/sql/CarbonDatasourceRelation.scala | 2 +
.../org/apache/spark/sql/CarbonSqlParser.scala | 56 ++++++++++----------
.../spark/sql/parser/CarbonSparkSqlParser.scala | 14 +++--
11 files changed, 149 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark/src/main/resources/data.csv
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/resources/data.csv b/examples/spark/src/main/resources/data.csv
index 76aec2d..5696978 100644
--- a/examples/spark/src/main/resources/data.csv
+++ b/examples/spark/src/main/resources/data.csv
@@ -1,11 +1,11 @@
-ID,date,country,name,phonetype,serialname,salary
-1,2015/7/23,china,aaa1,phone197,ASD69643,15000
-2,2015/7/24,china,aaa2,phone756,ASD42892,15001
-3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
-4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
-5,2015/7/27,china,aaa5,phone2441,ASD90633,15004
-6,2015/7/28,china,aaa6,phone294,ASD59961,15005
-7,2015/7/29,china,aaa7,phone610,ASD14875,15006
-8,2015/7/30,china,aaa8,phone1848,ASD57308,15007
-9,2015/7/18,china,aaa9,phone706,ASD86717,15008
-10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
\ No newline at end of file
+ID,date,country,name,phonetype,serialname,salary,floatField
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index 4dcc868..558139a 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -40,7 +40,7 @@ object AllDictionaryExample {
cc.sql("""
CREATE TABLE IF NOT EXISTS t3
(ID Int, date Timestamp, country String,
- name String, phonetype String, serialname String, salary Int)
+ name String, phonetype String, serialname String, salary Int,floatField float)
STORED BY 'carbondata'
""")
@@ -53,6 +53,10 @@ object AllDictionaryExample {
SELECT * FROM t3
""").show()
+ cc.sql("""
+ SELECT * FROM t3 where floatField=3.5
+ """).show()
+
cc.sql("DROP TABLE IF EXISTS t3")
// clean local dictionary files
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark2/src/main/resources/data.csv
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/resources/data.csv b/examples/spark2/src/main/resources/data.csv
index 2722edd..a9cdf9e 100644
--- a/examples/spark2/src/main/resources/data.csv
+++ b/examples/spark2/src/main/resources/data.csv
@@ -1,10 +1,10 @@
-1,10,100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23 11:01:01,aaa
-5,17,140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27 11:01:02,bbb
-1,11,100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23 11:01:03,ccc
-1,10,150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24 11:01:04,ddd
-1,10,100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23 11:01:05,eeee
-3,14,160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26 11:01:06,ff
-2,10,100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23 11:01:07,ggg
-1,10,100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23 11:01:08,hhh
-4,16,130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23 11:01:09,iii
-1,10,100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23 11:01:10,jjj
+1,10,100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23 11:01:01,aaa,2.5
+5,17,140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27 11:01:02,bbb,2.5
+1,11,100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23 11:01:03,ccc,2.5
+1,10,150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24 11:01:04,ddd,2.5
+1,10,100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23 11:01:05,eeee,3.5
+3,14,160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26 11:01:06,ff,2.5
+2,10,100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23 11:01:07,ggg,2.5
+1,10,100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23 11:01:08,hhh,2.5
+4,16,130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23 11:01:09,iii,2.5
+1,10,100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23 11:01:10,jjj,2.5
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 4923e5b..0d9c43f 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -75,7 +75,8 @@ object CarbonSessionExample {
| timestampField timestamp,
| decimalField decimal(18,2),
| dateField date,
- | charField char(5)
+ | charField char(5),
+ | floatField float
| )
| STORED BY 'carbondata'
| TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -88,7 +89,7 @@ object CarbonSessionExample {
s"""
| LOAD DATA LOCAL INPATH '$path'
| INTO TABLE carbon_table
- | options('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField')
+ | options('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
""".stripMargin)
// scalastyle:on
@@ -137,6 +138,12 @@ object CarbonSessionExample {
|where t1.stringField = t2.stringField
""".stripMargin).show
+ spark.sql("""
+ SELECT *
+ FROM carbon_table
+ where stringfield = 'spark' and floatField > 2.8
+ """).show
+
// Drop table
spark.sql("DROP TABLE IF EXISTS carbon_table")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark-common-test/src/test/resources/floatSample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/floatSample.csv b/integration/spark-common-test/src/test/resources/floatSample.csv
new file mode 100644
index 0000000..7cb46ba
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/floatSample.csv
@@ -0,0 +1,11 @@
+ID,date,country,name,phonetype,serialname,salary,rating
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000,2.34
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,2.34
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002,2.34
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003,2.34
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,2.34
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.5
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006,2.34
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,2.34
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008,2.34
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,2.34
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/FloatDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/FloatDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/FloatDataTypeTestCase.scala
new file mode 100644
index 0000000..41f3b40
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/primitiveTypes/FloatDataTypeTestCase.scala
@@ -0,0 +1,50 @@
+package org.apache.carbondata.integration.spark.testsuite.primitiveTypes
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for filter query on Float datatypes
+ *
+ * @author N00902756
+ *
+ */
+class FloatDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("DROP TABLE IF EXISTS tfloat")
+ sql("""
+ CREATE TABLE IF NOT EXISTS tfloat
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int,rating float)
+ STORED BY 'carbondata'
+ """)
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/floatSample.csv' into table tfloat
+ """)
+
+ }
+
+ test("select row whose rating is more than 2.8 from tfloat") {
+ checkAnswer(
+ sql("SELECT ID FROM tfloat where rating>2.8"),
+ Seq(Row(6)))
+ }
+
+ test("select row whose rating is 3.5 from tfloat") {
+ checkAnswer(
+ sql("SELECT ID FROM tfloat where rating=3.5"),
+ Seq(Row(6)))
+ }
+
+ test("select sum of rating column from tfloat") {
+ checkAnswer(
+ sql("SELECT sum(rating) FROM tfloat"),
+ Seq(Row(24.56)))
+ }
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS tfloat")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 570d8ed..0364371 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -31,6 +31,7 @@ object DataTypeConverterUtil {
case "bigint" => DataType.LONG
case "numeric" => DataType.DOUBLE
case "double" => DataType.DOUBLE
+ case "float" => DataType.DOUBLE
case "decimal" => DataType.DECIMAL
case "timestamp" => DataType.TIMESTAMP
case "date" => DataType.DATE
@@ -51,6 +52,7 @@ object DataTypeConverterUtil {
case "biginttype" => DataType.LONG
case "numerictype" => DataType.DOUBLE
case "doubletype" => DataType.DOUBLE
+ case "floattype" => DataType.DOUBLE
case "decimaltype" => DataType.DECIMAL
case "timestamptype" => DataType.TIMESTAMP
case "datetype" => DataType.DATE
@@ -67,6 +69,7 @@ object DataTypeConverterUtil {
case DataType.INT => "int"
case DataType.LONG => "bigint"
case DataType.DOUBLE => "double"
+ case DataType.FLOAT => "double"
case DataType.DECIMAL => "decimal"
case DataType.TIMESTAMP => "timestamp"
case DataType.DATE => "date"
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 1c1d5f1..c775d57 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -146,6 +146,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val NUMERIC = carbonKeyWord("NUMERIC")
protected val DECIMAL = carbonKeyWord("DECIMAL")
protected val DOUBLE = carbonKeyWord("DOUBLE")
+ protected val FLOAT = carbonKeyWord("FLOAT")
protected val SHORT = carbonKeyWord("SMALLINT")
protected val INT = carbonKeyWord("INT")
protected val BIGINT = carbonKeyWord("BIGINT")
@@ -818,7 +819,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
STRING ^^^ "string" | INTEGER ^^^ "integer" |
TIMESTAMP ^^^ "timestamp" | NUMERIC ^^^ "numeric" |
BIGINT ^^^ "bigint" | SHORT ^^^ "smallint" |
- INT ^^^ "int" | DOUBLE ^^^ "double" | decimalType | DATE ^^^ "date" | charType
+ INT ^^^ "int" | DOUBLE ^^^ "double" | FLOAT ^^^ "double" | decimalType |
+ DATE ^^^ "date" | charType
/**
* Matching the decimal(10,0) data type and returning the same.
@@ -895,6 +897,9 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
case "double" => Field(field.column, Some("Double"), field.name, Some(null), field.parent,
field.storeType, field.schemaOrdinal, field.precision, field.scale, field.rawSchema
)
+ case "float" => Field(field.column, Some("Double"), field.name, Some(null), field.parent,
+ field.storeType, field.schemaOrdinal, field.precision, field.scale, field.rawSchema
+ )
case "timestamp" =>
Field(field.column, Some("Timestamp"), field.name, Some(null),
field.parent, field.storeType, field.schemaOrdinal,
@@ -961,6 +966,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
case "Double" => Field(parentName + "." + field.column, Some("Double"),
Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
+ case "Float" => Field(parentName + "." + field.column, Some("Double"),
+ Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
case "Timestamp" => Field(parentName + "." + field.column, Some("Timestamp"),
Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
case "Numeric" => Field(parentName + "." + field.column, Some("Numeric"),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 5ea7129..67a0e3e 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -242,6 +242,7 @@ case class CarbonRelation(
.toLowerCase match {
case "int" => "long"
case "short" => "long"
+ case "float" => "double"
case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
case others => others
}),
@@ -271,6 +272,7 @@ case class CarbonRelation(
.toLowerCase match {
case "int" => "long"
case "short" => "long"
+ case "float" => "double"
case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
.getColumnSchema.getScale + ")"
case others => others
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index e9738cb..532918d 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -121,22 +121,22 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
* For handling the create table DDl systax compatible to Hive syntax
*/
protected lazy val createTable: Parser[LogicalPlan] =
- restInput ^^ {
-
- case statement =>
- try {
- // DDl will be parsed and we get the AST tree from the HiveQl
- val node = HiveQlWrapper.getAst(statement)
- // processing the AST tree
- nodeToPlan(node)
- } catch {
- // MalformedCarbonCommandException need to be throw directly, parser will catch it
- case ce: MalformedCarbonCommandException =>
- throw ce
- case e: Exception =>
- sys.error("Parsing error") // no need to do anything.
- }
- }
+ restInput ^^ {
+
+ case statement =>
+ try {
+ // DDl will be parsed and we get the AST tree from the HiveQl
+ val node = HiveQlWrapper.getAst(statement)
+ // processing the AST tree
+ nodeToPlan(node)
+ } catch {
+ // MalformedCarbonCommandException need to be throw directly, parser will catch it
+ case ce: MalformedCarbonCommandException =>
+ throw ce
+ case e: Exception =>
+ sys.error("Parsing error") // no need to do anything.
+ }
+ }
/**
* This function will traverse the tree and logical plan will be formed using that.
@@ -211,8 +211,10 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
f.scale = scale
f.dataType = Some("decimal")
}
- if(f.dataType.getOrElse("").startsWith("char")) {
+ if (f.dataType.getOrElse("").startsWith("char")) {
f.dataType = Some("char")
+ } else if (f.dataType.getOrElse("").startsWith("float")) {
+ f.dataType = Some("float")
}
f.rawSchema = x
fields ++= Seq(f)
@@ -254,7 +256,7 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
case Token("TOK_LIKETABLE", child :: Nil) =>
likeTableName = child.getChild(0).getText()
case Token("TOK_ALTERTABLE_BUCKETS",
- Token("TOK_TABCOLNAME", list)::numberOfBuckets) =>
+ Token("TOK_TABCOLNAME", list) :: numberOfBuckets) =>
val cols = list.map(_.getText)
if (cols != null) {
bucketFields = Some(BucketFields(cols,
@@ -417,13 +419,13 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
}
protected lazy val deleteRecords: Parser[LogicalPlan] =
- (DELETE ~> FROM ~> table) ~ (WHERE ~> restInput).? <~ opt(";") ^^ {
+ (DELETE ~> FROM ~> table) ~ (WHERE ~> restInput).? <~ opt(";") ^^ {
case table ~ condition =>
val tableName = getTableName(table.tableIdentifier)
val alias = table.alias.getOrElse("")
val stmt = condition match {
case Some(cond) =>
- "select tupleId from " + tableName + " " + alias + " where " + cond
+ "select tupleId from " + tableName + " " + alias + " where " + cond
case _ =>
"select tupleId from " + tableName + " " + alias
}
@@ -432,10 +434,10 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
protected lazy val updateTable: Parser[LogicalPlan] =
UPDATE ~> table ~
- (SET ~> "(" ~> repsep(element, ",") <~ ")") ~
- ( "=" ~> restInput ) <~ opt(";") ^^ {
- case tab~ columns ~ rest =>
- val (sel, where ) = splitQuery(rest)
+ (SET ~> "(" ~> repsep(element, ",") <~ ")") ~
+ ("=" ~> restInput) <~ opt(";") ^^ {
+ case tab ~ columns ~ rest =>
+ val (sel, where) = splitQuery(rest)
val (selectStmt, relation) =
if (!sel.toLowerCase.startsWith("select ")) {
if (sel.trim.isEmpty) {
@@ -483,7 +485,7 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
} else if (ch == '\"') {
foundDoubleQuotes = !foundDoubleQuotes
}
- else if (ch == '(' && !foundSingleQuotes && !foundDoubleQuotes) {
+ else if (ch == '(' && !foundSingleQuotes && !foundDoubleQuotes) {
bracketCount = bracketCount + 1
stack.push(ch)
} else if (ch == ')' && !foundSingleQuotes && !foundDoubleQuotes) {
@@ -507,7 +509,7 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
protected lazy val table: Parser[UnresolvedRelation] = {
- rep1sep(attributeName, ".") ~ opt(ident)^^ {
+ rep1sep(attributeName, ".") ~ opt(ident) ^^ {
case tableIdent ~ alias => UnresolvedRelation(tableIdent, alias)
}
}
@@ -541,7 +543,7 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
}
protected lazy val element: Parser[String] =
- (ident <~ ".").? ~ ident ^^ {
+ (ident <~ ".").? ~ ident ^^ {
case table ~ column => column.toLowerCase
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/54e83045/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index 04de23d..3bcf165 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -21,9 +21,9 @@ import scala.collection.mutable
import org.apache.spark.sql.catalyst.parser.{AbstractSqlParser, ParseException, SqlBaseParser}
import org.apache.spark.sql.catalyst.parser.ParserUtils._
-import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{CreateTableContext, TablePropertyListContext}
+import org.apache.spark.sql.catalyst.parser.SqlBaseParser.{CreateTableContext,
+TablePropertyListContext}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.trees.Origin
import org.apache.spark.sql.execution.SparkSqlAstBuilder
import org.apache.spark.sql.execution.command.{BucketFields, CreateTable, Field, TableModel}
import org.apache.spark.sql.internal.{SQLConf, VariableSubstitution}
@@ -108,7 +108,12 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
val schema = cols ++ partitionCols
val fields = schema.map { col =>
- val x = col.name + ' ' + col.dataType.catalogString
+ val x = if (col.dataType.catalogString == "float") {
+ col.name + " double"
+ }
+ else {
+ col.name + ' ' + col.dataType.catalogString
+ }
val f: Field = parser.anyFieldDef(new parser.lexical.Scanner(x))
match {
case parser.Success(field, _) => field.asInstanceOf[Field]
@@ -127,6 +132,9 @@ class CarbonSqlAstBuilder(conf: SQLConf) extends SparkSqlAstBuilder(conf) {
if (f.dataType.getOrElse("").startsWith("char")) {
f.dataType = Some("char")
}
+ else if (f.dataType.getOrElse("").startsWith("float")) {
+ f.dataType = Some("double")
+ }
f.rawSchema = x
f
}
[2/2] incubator-carbondata git commit: [CARBONDATA-390] Support for
float datatype This closes #521
Posted by ra...@apache.org.
[CARBONDATA-390] Support for float datatype This closes #521
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b5c20a80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b5c20a80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b5c20a80
Branch: refs/heads/master
Commit: b5c20a80aced23b94177e8c93ea511bc68e92ac1
Parents: 396d0d6 54e8304
Author: ravipesala <ra...@gmail.com>
Authored: Mon Jan 16 16:56:48 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Jan 16 16:56:48 2017 +0530
----------------------------------------------------------------------
examples/spark/src/main/resources/data.csv | 22 ++++----
.../examples/AllDictionaryExample.scala | 6 ++-
examples/spark2/src/main/resources/data.csv | 20 +++----
.../examples/CarbonSessionExample.scala | 11 +++-
.../src/test/resources/floatSample.csv | 11 ++++
.../primitiveTypes/FloatDataTypeTestCase.scala | 50 +++++++++++++++++
.../spark/util/DataTypeConverterUtil.scala | 3 ++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 9 +++-
.../spark/sql/CarbonDatasourceRelation.scala | 2 +
.../org/apache/spark/sql/CarbonSqlParser.scala | 56 ++++++++++----------
.../spark/sql/parser/CarbonSparkSqlParser.scala | 14 +++--
11 files changed, 149 insertions(+), 55 deletions(-)
----------------------------------------------------------------------