You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/20 07:29:33 UTC
[27/56] [abbrv] carbondata git commit: fixed CARBONDATA-1155
fixed CARBONDATA-1155
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3c675818
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3c675818
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3c675818
Branch: refs/heads/streaming_ingest
Commit: 3c67581825ea8149927e61661388d839e09ab5fd
Parents: ecc0203
Author: rahulforallp <ra...@knoldus.in>
Authored: Mon Jun 12 16:27:40 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Jun 15 17:32:15 2017 +0530
----------------------------------------------------------------------
.../testsuite/sortcolumns/TestSortColumns.scala | 34 ++++++++++++++++++++
.../store/writer/AbstractFactDataWriter.java | 4 ++-
2 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c675818/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index f08cba2..01c8021 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -38,6 +38,39 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select empno from sorttable1"), sql("select empno from sorttable1 order by empno"))
}
+ test("create table with no dictionary sort_columns with dictionary exclude") {
+ sql(
+ "CREATE TABLE sorttable1a (empno String, empname String, designation String, doj Timestamp," +
+ " workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
+ "projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int," +
+ "utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties" +
+ "('dictionary_exclude'='empno','sort_columns'='empno')")
+ sql(
+ s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1a OPTIONS
+ |('DELIMITER'= ',', 'QUOTECHAR'= '\"','SORT_SCOPE'='BATCH_SORT',
+ |'batch_sort_size_inmb'='64')""".stripMargin)
+ checkAnswer(sql("select empname from sorttable1a"),
+ sql("select empname from origintable1 order by empname"))
+ }
+
+ test(
+ "create table with no dictionary sort_columns where NumberOfNoDictSortColumns < " +
+ "NoDictionaryCount")
+ {
+ sql(
+ "CREATE TABLE sorttable1b (empno String, empname String, designation String, doj Timestamp," +
+ " workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
+ "projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int," +
+ "utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties" +
+ "('dictionary_exclude'='empno,empname,workgroupcategoryname','sort_columns'='empno,empname')")
+ sql(
+ s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1b OPTIONS
+ |('DELIMITER'= ',', 'QUOTECHAR'= '\"','SORT_SCOPE'='BATCH_SORT',
+ |'batch_sort_size_inmb'='64')""".stripMargin)
+ checkAnswer(sql("select empname from sorttable1b"),
+ sql("select empname from origintable1 order by empname"))
+ }
+
test("create table with dictionary sort_columns") {
sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -268,6 +301,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists origintable2")
sql("drop table if exists sorttable1")
sql("drop table if exists sorttableDesc")
+ sql("drop table if exists sorttable1a")
sql("drop table if exists sorttable2")
sql("drop table if exists sorttable3")
sql("drop table if exists sorttable4_offheap_safe")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3c675818/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index aaeaf66..5d9e081 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -603,7 +603,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
short numberOfByteToStorLength = 2;
// as length part is removed, so each no dictionary value index
// needs to be reshuffled by 2 bytes
- for (int i = 0; i < dataWriterVo.getNoDictionaryCount(); i++) {
+ int NumberOfNoDictSortColumns =
+ dataWriterVo.getSegmentProperties().getNumberOfNoDictSortColumns();
+ for (int i = 0; i < NumberOfNoDictSortColumns; i++) {
output.putShort((short) (buffer.getShort() - numberOfByteToStorLength));
}
// copy the data part