You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/04/21 06:34:33 UTC
[1/2] incubator-carbondata git commit: changes done: 1. Corrected
temp store location path formation for compaction through sort step. 2.
Correct filter query bug on datatype changed for a decimal column. 3.
Corrected data type change validations. New pr
Repository: incubator-carbondata
Updated Branches:
refs/heads/master c4af8afac -> 5c69c4918
changes done:
1. Corrected temp store location path formation for compaction through sort step.
2. Correct filter query bug on datatype changed for a decimal column.
3. Corrected data type change validations. New precision should always be greater than existing precision and new scale value can be same as old scale value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f890d008
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f890d008
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f890d008
Branch: refs/heads/master
Commit: f890d0082716f2f5adb1aa823391dd6332986f42
Parents: c4af8af
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Apr 21 00:23:31 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Apr 21 12:02:38 2017 +0530
----------------------------------------------------------------------
.../vector/MeasureDataVectorProcessor.java | 3 +++
.../carbondata/spark/load/CarbonLoaderUtil.java | 8 +++---
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 10 +++----
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 +--
.../AlterTableValidationTestCase.scala | 11 ++++++++
.../vectorreader/AddColumnTestCases.scala | 14 ++++++++++
.../merger/CompactionResultSortProcessor.java | 4 +--
.../newflow/DataLoadProcessBuilder.java | 4 +--
.../sortandgroupby/sortdata/SortParameters.java | 6 +++--
.../util/CarbonDataProcessorUtil.java | 28 +++++++++++++++-----
10 files changed, 68 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index 3c65198..c100ee2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -251,6 +251,9 @@ public class MeasureDataVectorProcessor {
} else {
BigDecimal decimal =
dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(currentRow);
+ if (info.measure.getMeasure().getScale() > decimal.scale()) {
+ decimal = decimal.setScale(info.measure.getMeasure().getScale());
+ }
Decimal toDecimal = Decimal.apply(decimal);
vector.putDecimal(vectorOffset, toDecimal, precision);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 27a8e3a..4806a93 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -73,6 +73,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.merger.NodeBlockRelation;
import org.apache.carbondata.processing.merger.NodeMultiBlockRelation;
import org.apache.carbondata.processing.model.CarbonLoadModel;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import com.google.gson.Gson;
import org.apache.spark.SparkConf;
@@ -168,11 +169,8 @@ public final class CarbonLoaderUtil {
boolean isCompactionFlow) {
String databaseName = loadModel.getDatabaseName();
String tableName = loadModel.getTableName();
- String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
- + CarbonCommonConstants.UNDERSCORE + loadModel.getTaskNo();
- if (isCompactionFlow) {
- tempLocationKey = CarbonCommonConstants.COMPACTION_KEY_WORD + '_' + tempLocationKey;
- }
+ String tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(databaseName, tableName, loadModel.getTaskNo(), isCompactionFlow);
// form local store location
final String localStoreLocation = CarbonProperties.getInstance()
.getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 49984c9..4ebbf60 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -79,11 +79,11 @@ class CarbonMergerRDD[K, V](
val iter = new Iterator[(K, V)] {
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
- val tempLocationKey: String = CarbonCommonConstants
- .COMPACTION_KEY_WORD + '_' + carbonLoadModel
- .getDatabaseName + '_' + carbonLoadModel
- .getTableName + '_' + carbonLoadModel.getTaskNo
-
+ val tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
+ carbonLoadModel.getTableName,
+ carbonLoadModel.getTaskNo,
+ true)
// this property is used to determine whether temp location for carbon is inside
// container temp dir or is yarn application directory.
val carbonUseLocalDir = CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index c95b7ea..c88f7a4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -177,11 +177,11 @@ object CarbonScalaUtil {
} cannot be modified. Specified precision value ${
dataTypeInfo
.precision
- } should be greater or equal to current precision value ${
+ } should be greater than current precision value ${
carbonColumn.getColumnSchema
.getPrecision
}")
- } else if (dataTypeInfo.scale <= carbonColumn.getColumnSchema.getScale) {
+ } else if (dataTypeInfo.scale < carbonColumn.getColumnSchema.getScale) {
sys
.error(s"Given column ${
carbonColumn
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index c826b0c..93b57c2 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -293,6 +293,17 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql("alter table default.restructure change intfield intField bigint")
checkExistence(sql("desc restructure"), true, "intfieldbigint")
sql("alter table default.restructure change decimalfield deciMalfield Decimal(11,3)")
+ sql("alter table default.restructure change decimalfield deciMalfield Decimal(12,3)")
+ intercept[RuntimeException] {
+ sql("alter table default.restructure change decimalfield deciMalfield Decimal(12,3)")
+ }
+ intercept[RuntimeException] {
+ sql("alter table default.restructure change decimalfield deciMalfield Decimal(13,1)")
+ }
+ intercept[RuntimeException] {
+ sql("alter table default.restructure change decimalfield deciMalfield Decimal(13,5)")
+ }
+ sql("alter table default.restructure change decimalfield deciMalfield Decimal(13,4)")
}
test("test change datatype of string to int column") {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index 3540900..e341f27 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -294,6 +294,20 @@ class AddColumnTestCases extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon_dictionary_is_null")
}
+ test("test add column for new decimal column filter query") {
+ sql("DROP TABLE IF EXISTS alter_decimal_filter")
+ sql(
+ "create table alter_decimal_filter (n1 string, n2 int, n3 decimal(3,2)) stored by " +
+ "'carbondata'")
+ sql("insert into alter_decimal_filter select 'xx',1,1.22")
+ sql("insert into alter_decimal_filter select 'xx',1,1.23")
+ sql("alter table alter_decimal_filter change n3 n3 decimal(8,4)")
+ sql("insert into alter_decimal_filter select 'dd',2,111.111")
+ sql("select * from alter_decimal_filter where n3 = 1.22").show()
+ checkAnswer(sql("select * from alter_decimal_filter where n3 = 1.22"),
+ Row("xx", 1, new BigDecimal(1.2200).setScale(4, RoundingMode.HALF_UP)))
+ sql("DROP TABLE IF EXISTS alter_decimal_filter")
+ }
override def afterAll {
sql("DROP TABLE IF EXISTS addcolumntest")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 8c1f577..a76cc0a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -348,7 +348,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
.createSortParameters(carbonLoadModel.getDatabaseName(), tableName, dimensionColumnCount,
segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount,
carbonLoadModel.getPartitionId(), segmentId, carbonLoadModel.getTaskNo(),
- noDictionaryColMapping);
+ noDictionaryColMapping, true);
return parameters;
}
@@ -390,7 +390,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
private void initTempStoreLocation() {
tempStoreLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(carbonLoadModel.getDatabaseName(), tableName,
- carbonLoadModel.getTaskNo(), carbonLoadModel.getPartitionId(), segmentId, false);
+ carbonLoadModel.getTaskNo(), carbonLoadModel.getPartitionId(), segmentId, true);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 3980bcf..8865518 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -129,8 +129,8 @@ public final class DataLoadProcessBuilder {
CarbonDataLoadConfiguration configuration = new CarbonDataLoadConfiguration();
String databaseName = loadModel.getDatabaseName();
String tableName = loadModel.getTableName();
- String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
- + CarbonCommonConstants.UNDERSCORE + loadModel.getTaskNo();
+ String tempLocationKey = CarbonDataProcessorUtil
+ .getTempStoreLocationKey(databaseName, tableName, loadModel.getTaskNo(), false);
CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation);
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.STORE_LOCATION_HDFS, loadModel.getStorePath());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index 41e3018..3c3a9d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -423,7 +423,8 @@ public class SortParameters {
public static SortParameters createSortParameters(String databaseName, String tableName,
int dimColCount, int complexDimColCount, int measureColCount, int noDictionaryCount,
- String partitionID, String segmentId, String taskNo, boolean[] noDictionaryColMaping) {
+ String partitionID, String segmentId, String taskNo, boolean[] noDictionaryColMaping,
+ boolean isCompactionFlow) {
SortParameters parameters = new SortParameters();
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(databaseName);
@@ -458,7 +459,8 @@ public class SortParameters {
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
String carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId, false);
+ .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId,
+ isCompactionFlow);
parameters.setTempFileLocation(
carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
LOGGER.info("temp file location" + parameters.getTempFileLocation());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f890d008/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 178b43b..41bfbed 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -193,12 +193,8 @@ public final class CarbonDataProcessorUtil {
*/
public static String getLocalDataFolderLocation(String databaseName, String tableName,
String taskId, String partitionId, String segmentId, boolean isCompactionFlow) {
- String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
- + CarbonCommonConstants.UNDERSCORE + taskId;
- if (isCompactionFlow) {
- tempLocationKey = CarbonCommonConstants.COMPACTION_KEY_WORD + '_' + tempLocationKey;
- }
-
+ String tempLocationKey =
+ getTempStoreLocationKey(databaseName, tableName, taskId, isCompactionFlow);
String baseStorePath = CarbonProperties.getInstance()
.getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
CarbonTable carbonTable = CarbonMetadata.getInstance()
@@ -212,6 +208,26 @@ public final class CarbonDataProcessorUtil {
}
/**
+ * This method will form the key for getting the temporary location set in carbon properties
+ *
+ * @param databaseName
+ * @param tableName
+ * @param taskId
+ * @param isCompactionFlow
+ * @return
+ */
+ public static String getTempStoreLocationKey(String databaseName, String tableName, String taskId,
+ boolean isCompactionFlow) {
+ String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
+ + CarbonCommonConstants.UNDERSCORE + taskId;
+ if (isCompactionFlow) {
+ tempLocationKey = CarbonCommonConstants.COMPACTION_KEY_WORD + CarbonCommonConstants.UNDERSCORE
+ + tempLocationKey;
+ }
+ return tempLocationKey;
+ }
+
+ /**
* Preparing the boolean [] to map whether the dimension is no Dictionary or not.
*/
public static boolean[] getNoDictionaryMapping(DataField[] fields) {
[2/2] incubator-carbondata git commit: [CARBONDATA-968] Alter temp
store location and decimal data type incorrect result display correction.
This closes #829
Posted by ra...@apache.org.
[CARBONDATA-968] Alter temp store location and decimal data type incorrect result display correction. This closes #829
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5c69c491
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5c69c491
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5c69c491
Branch: refs/heads/master
Commit: 5c69c49189e6e9582cf98e67160092e593cfa2c2
Parents: c4af8af f890d00
Author: ravipesala <ra...@gmail.com>
Authored: Fri Apr 21 12:03:47 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Apr 21 12:03:47 2017 +0530
----------------------------------------------------------------------
.../vector/MeasureDataVectorProcessor.java | 3 +++
.../carbondata/spark/load/CarbonLoaderUtil.java | 8 +++---
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 10 +++----
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 +--
.../AlterTableValidationTestCase.scala | 11 ++++++++
.../vectorreader/AddColumnTestCases.scala | 14 ++++++++++
.../merger/CompactionResultSortProcessor.java | 4 +--
.../newflow/DataLoadProcessBuilder.java | 4 +--
.../sortandgroupby/sortdata/SortParameters.java | 6 +++--
.../util/CarbonDataProcessorUtil.java | 28 +++++++++++++++-----
10 files changed, 68 insertions(+), 24 deletions(-)
----------------------------------------------------------------------