You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2019/12/28 02:16:14 UTC
[carbondata] branch master updated: [CARBONDATA-3626] Improve
performance when load data into carbon table with lots of columns
This is an automated email from the ASF dual-hosted git repository.
jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 431d5fb [CARBONDATA-3626] Improve performance when load data into carbon table with lots of columns
431d5fb is described below
commit 431d5fb1b0bfe9974eddc0984dae9c3abaaa3a9c
Author: 沈洪 <yu...@alipay.com>
AuthorDate: Sat Dec 21 21:32:34 2019 +0800
[CARBONDATA-3626] Improve performance when load data into carbon table with lots of columns
This closes #3525
---
.../spark/sql/execution/datasources/SparkCarbonTableFormat.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
index 99a9981..d1379aa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala
@@ -353,7 +353,8 @@ private class CarbonOutputWriter(path: String,
def writeCarbon(row: InternalRow): Unit = {
val data = new Array[AnyRef](fieldTypes.length + partitionData.length)
var i = 0
- while (i < fieldTypes.length) {
+ val fieldTypesLen = fieldTypes.length
+ while (i < fieldTypesLen) {
if (!row.isNullAt(i)) {
fieldTypes(i) match {
case StringType =>
@@ -367,7 +368,7 @@ private class CarbonOutputWriter(path: String,
i += 1
}
if (partitionData.length > 0) {
- System.arraycopy(partitionData, 0, data, fieldTypes.length, partitionData.length)
+ System.arraycopy(partitionData, 0, data, fieldTypesLen, partitionData.length)
}
writable.set(data)
recordWriter.write(NullWritable.get(), writable)