You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/15 08:51:23 UTC
[1/2] carbondata git commit: Don't reuse the converRow.
Repository: carbondata
Updated Branches:
refs/heads/master 4319be8b7 -> ecc0203de
Don't reuse the converRow.
Fix bugs on spark-1.6
Add flag.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6c41f891
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6c41f891
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6c41f891
Branch: refs/heads/master
Commit: 6c41f891ddd4b12193d6a38b326b04c1e1251342
Parents: 4319be8
Author: Yadong Qi <qi...@gmail.com>
Authored: Thu Jun 15 15:16:28 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Jun 15 16:50:50 2017 +0800
----------------------------------------------------------------------
.../dataload/TestGlobalSortDataLoad.scala | 17 ++++++++--------
.../load/DataLoadProcessorStepOnSpark.scala | 3 ++-
.../newflow/sort/SortStepRowUtil.java | 21 ++++++++++++--------
.../UnsafeSingleThreadFinalSortFilesMerger.java | 2 +-
4 files changed, 25 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c41f891/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index 190b7df..2842a16 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -233,11 +233,11 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
| STORED BY 'org.apache.carbondata.format'
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_localsort_delete")
- sql("DELETE FROM carbon_localsort_delete WHERE id = 1")
+ sql("DELETE FROM carbon_localsort_delete WHERE id = 1").show
sql(s"LOAD DATA LOCAL INPATH '$filePath' INTO TABLE carbon_globalsort " +
"OPTIONS('SORT_SCOPE'='GLOBAL_SORT')")
- sql("DELETE FROM carbon_globalsort WHERE id = 1")
+ sql("DELETE FROM carbon_globalsort WHERE id = 1").show
assert(getIndexFileCount("carbon_globalsort") === 3)
checkAnswer(sql("SELECT COUNT(*) FROM carbon_globalsort"), Seq(Row(11)))
@@ -280,12 +280,13 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
test("Test with different date types") {
val path = s"$projectPath/examples/spark2/src/main/resources/data.csv"
+ sql("DROP TABLE IF EXISTS carbon_localsort_difftypes")
sql(
s"""
| CREATE TABLE carbon_localsort_difftypes(
- | shortField SHORT,
+ | shortField smallint,
| intField INT,
- | bigintField LONG,
+ | bigintField bigint,
| doubleField DOUBLE,
| stringField STRING,
| timestampField TIMESTAMP,
@@ -299,16 +300,16 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with Befo
sql(
s"""
| LOAD DATA LOCAL INPATH '$path' INTO TABLE carbon_localsort_difftypes
- | OPTIONS('SORT_SCOPE'='GLOBAL_SORT',
- | 'FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
+ | OPTIONS('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
""".stripMargin)
+ sql("DROP TABLE IF EXISTS carbon_globalsort_difftypes")
sql(
s"""
| CREATE TABLE carbon_globalsort_difftypes(
- | shortField SHORT,
+ | shortField smallint,
| intField INT,
- | bigintField LONG,
+ | bigintField bigint,
| doubleField DOUBLE,
| stringField STRING,
| timestampField TIMESTAMP,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c41f891/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index fb9223c..47253b5 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -135,7 +135,8 @@ object DataLoadProcessorStepOnSpark {
override def hasNext: Boolean = rows.hasNext
override def next(): CarbonRow = {
- val row = new CarbonRow(SortStepRowUtil.convertRow(rows.next().getData, sortParameters))
+ val row =
+ new CarbonRow(SortStepRowUtil.convertRow(rows.next().getData, sortParameters, true))
rowCounter.add(1)
row
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c41f891/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 53f1497..5b0685b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -25,7 +25,8 @@ import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
import org.apache.carbondata.processing.util.NonDictionaryUtil;
public class SortStepRowUtil {
- public static Object[] convertRow(Object[] data, SortParameters parameters) {
+ public static Object[] convertRow(Object[] data, SortParameters parameters,
+ boolean needConvertDecimalToByte) {
int measureCount = parameters.getMeasureColCount();
int dimensionCount = parameters.getDimColCount();
int complexDimensionCount = parameters.getComplexDimColCount();
@@ -62,16 +63,20 @@ public class SortStepRowUtil {
DataType[] measureDataType = parameters.getMeasureDataType();
// read measure values
for (int i = 0; i < measureCount; i++) {
- Object value = data[allCount];
- if (null != value) {
- if (measureDataType[i] == DataType.DECIMAL) {
- BigDecimal decimal = (BigDecimal) value;
- measures[index++] = DataTypeUtil.bigDecimalToByte(decimal);
+ if (needConvertDecimalToByte) {
+ Object value = data[allCount];
+ if (null != value) {
+ if (measureDataType[i] == DataType.DECIMAL) {
+ BigDecimal decimal = (BigDecimal) value;
+ measures[index++] = DataTypeUtil.bigDecimalToByte(decimal);
+ } else {
+ measures[index++] = value;
+ }
} else {
- measures[index++] = value;
+ measures[index++] = null;
}
} else {
- measures[index++] = null;
+ measures[index++] = data[allCount];
}
allCount++;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6c41f891/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index 7b8ec79..acb976f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -202,7 +202,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
* @return sorted row
*/
public Object[] next() {
- return SortStepRowUtil.convertRow(getSortedRecordFromFile(), parameters);
+ return SortStepRowUtil.convertRow(getSortedRecordFromFile(), parameters, false);
}
/**
[2/2] carbondata git commit: [CARBONDATA-1163][Bug-Fix] Fix CI
failures This closes #1037
Posted by ja...@apache.org.
[CARBONDATA-1163][Bug-Fix] Fix CI failures This closes #1037
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ecc0203d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ecc0203d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ecc0203d
Branch: refs/heads/master
Commit: ecc0203de9dc60801de595720ee7dc148d5b478d
Parents: 4319be8 6c41f89
Author: jackylk <ja...@huawei.com>
Authored: Thu Jun 15 16:51:08 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Jun 15 16:51:08 2017 +0800
----------------------------------------------------------------------
.../dataload/TestGlobalSortDataLoad.scala | 17 ++++++++--------
.../load/DataLoadProcessorStepOnSpark.scala | 3 ++-
.../newflow/sort/SortStepRowUtil.java | 21 ++++++++++++--------
.../UnsafeSingleThreadFinalSortFilesMerger.java | 2 +-
4 files changed, 25 insertions(+), 18 deletions(-)
----------------------------------------------------------------------