You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2017/05/09 07:21:34 UTC
[1/2] carbondata git commit: [CARBONDATA-1013] Changes to adhere Join
without dictionary column
Repository: carbondata
Updated Branches:
refs/heads/master b16ab636c -> 393cd6fcf
[CARBONDATA-1013] Changes to adhere Join without dictionary column
[CARBONDATA-1013] Changes to adhere join condition on dictionary columns
Changes to adhere Join without Dictionary Columns
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8024a49b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8024a49b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8024a49b
Branch: refs/heads/master
Commit: 8024a49b475041f14c82be91a16889e2e39e7a63
Parents: b16ab63
Author: SRIGOPALMOHANTY <SR...@GMAIL.COM>
Authored: Mon May 8 13:13:16 2017 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue May 9 15:20:25 2017 +0800
----------------------------------------------------------------------
.../joinquery/JoinWithoutDictionaryColumn.scala | 100 +++++++++++++++++++
.../sql/optimizer/CarbonLateDecodeRule.scala | 23 +++--
2 files changed, 112 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8024a49b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
new file mode 100644
index 0000000..d726858
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.joinquery
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test cases for testing columns having \N or \null values for non numeric columns
+ */
+class JoinWithoutDictionaryColumn extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("drop table if exists mobile")
+ sql("drop table if exists emp")
+
+ sql("drop table if exists mobile_d")
+ sql("drop table if exists emp_d")
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+ )
+
+ sql(
+ """
+ create table mobile (mid String,Mobileid String, Color String, id int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color')
+ """)
+ sql(
+ """
+ create table emp (eid String,ename String, Mobileid String,Color String, id int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color')
+ """)
+
+ sql(
+ """
+ create table mobile_d (mid String,Mobileid String, Color String, id int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color','DICTIONARY_INCLUDE'='Mobileid')
+ """)
+ sql(
+ """
+ create table emp_d (eid String,ename String, Mobileid String,Color String, id int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color','DICTIONARY_INCLUDE'='Mobileid')
+ """)
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/join/mobile.csv' into table
+ mobile
+ OPTIONS('FILEHEADER'='mid,Mobileid,Color,id')
+ """)
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/join/employee.csv' into table
+ emp
+ OPTIONS('FILEHEADER'='eid,ename,Mobileid,Color,id')
+ """)
+
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/join/mobile.csv' into table
+ mobile_d
+ OPTIONS('FILEHEADER'='mid,Mobileid,Color,id')
+ """)
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/join/employee.csv' into table
+ emp_d
+ OPTIONS('FILEHEADER'='eid,ename,Mobileid,Color,id')
+ """)
+ }
+
+ test("select * from emp join mobile on emp.Mobileid=mobile.Mobileid") {
+ checkAnswer(
+ sql("select * from emp join mobile on emp.Mobileid=mobile.Mobileid"),
+ sql("select * from emp_d join mobile_d on emp_d.Mobileid=mobile_d.Mobileid")
+ )
+ }
+
+ override def afterAll {
+ sql("drop table if exists emp")
+ sql("drop table if exists mobile")
+ sql("drop table if exists emp_d")
+ sql("drop table if exists mobile_d")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8024a49b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 45cc330..aff34ea 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -321,7 +321,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper {
Filter(filter.condition, child)
}
- case j: Join
+ case j: Join
if !(j.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
j.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
val attrsOnJoin = new util.HashSet[Attribute]
@@ -337,7 +337,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper {
val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
- if (attrsOnJoin.size() > 0) {
+ val join = if (attrsOnJoin.size() > 0) {
attrsOnJoin.asScala.map { attr =>
if (qualifierPresence(j.left, attr)) {
@@ -361,18 +361,19 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper {
new util.HashSet[AttributeReferenceWrapper](),
j.right)
}
- if (!decoder) {
- decoder = true
- CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
- new util.HashSet[AttributeReferenceWrapper](),
- Join(leftPlan, rightPlan, j.joinType, j.condition),
- isOuter = true)
- } else {
- Join(leftPlan, rightPlan, j.joinType, j.condition)
- }
+ Join(leftPlan, rightPlan, j.joinType, j.condition)
} else {
j
}
+ if (!decoder) {
+ decoder = true
+ CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
+ new util.HashSet[AttributeReferenceWrapper](),
+ join,
+ isOuter = true)
+ } else {
+ join
+ }
case p: Project
if relations.nonEmpty && !p.child.isInstanceOf[CarbonDictionaryTempDecoder] =>
[2/2] carbondata git commit: [CARBONDATA-1013] Changes to adhere Join
without dictionary column This closes #891
Posted by ch...@apache.org.
[CARBONDATA-1013] Changes to adhere Join without dictionary column This closes #891
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/393cd6fc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/393cd6fc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/393cd6fc
Branch: refs/heads/master
Commit: 393cd6fcf777e72b3d71e092bbf4bc4b6d2b9c2e
Parents: b16ab63 8024a49
Author: chenliang613 <ch...@huawei.com>
Authored: Tue May 9 15:21:22 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue May 9 15:21:22 2017 +0800
----------------------------------------------------------------------
.../joinquery/JoinWithoutDictionaryColumn.scala | 100 +++++++++++++++++++
.../sql/optimizer/CarbonLateDecodeRule.scala | 23 +++--
2 files changed, 112 insertions(+), 11 deletions(-)
----------------------------------------------------------------------