You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/01 10:04:59 UTC
[01/47] incubator-carbondata git commit: [CARBONDATA-79] When
timestamp is in the middle of other complex types, data load is failing (#841)
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 10ed89ab5 -> 80cfa39c6
[CARBONDATA-79] When timestamp is in the middle of other complex types, data load is failing (#841)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a326bb8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a326bb8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a326bb8f
Branch: refs/heads/master
Commit: a326bb8f7e71a6900fd977e9f3b63998d0abca2d
Parents: ad48c4c
Author: nareshpr <pr...@gmail.com>
Authored: Tue Jul 19 16:57:39 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Tue Jul 19 16:57:39 2016 +0530
----------------------------------------------------------------------
integration/spark/src/test/resources/timestampdata.csv | 2 ++
.../TestComplexPrimitiveTimestampDirectDictionary.scala | 12 ++++++++++++
.../csvbased/CarbonCSVBasedDimSurrogateKeyGen.java | 2 ++
3 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/integration/spark/src/test/resources/timestampdata.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/timestampdata.csv b/integration/spark/src/test/resources/timestampdata.csv
new file mode 100644
index 0000000..b0339bc
--- /dev/null
+++ b/integration/spark/src/test/resources/timestampdata.csv
@@ -0,0 +1,2 @@
+imei001,rat$aaa1,111$111,2015-01-01 13:00:00.000$2015-01-01 13:00:00.000,16$64,babu$001
+imei002,rat$aaa2,111$112,2015-01-01 13:00:00.000$2015-01-02 13:00:00.000,16$65,babu$002
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
index 43bf7ea..611a1ec 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala
@@ -53,6 +53,18 @@ class TestComplexPrimitiveTimestampDirectDictionary extends QueryTest with Befor
sql("select * from complexhivetimestamptable"))
}
+ test("timestamp complex type in the middle of complex types") {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss.SSS")
+ sql("CREATE TABLE testtimestampcarbon(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA local inpath './src/test/resources/timestampdata.csv' INTO TABLE testtimestampcarbon options('DELIMITER'=',', 'QUOTECHAR'='\"','COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'='imei,rat,sid,end_time,probeid,contact')")
+ sql("CREATE TABLE testtimestamphive(imei string,rat array<string>, sid array<int>, end_time array<Timestamp>, probeid array<double>, contact struct<name:string, id:string>)row format delimited fields terminated by ',' collection items terminated by '$'")
+ sql("LOAD DATA local inpath './src/test/resources/timestampdata.csv' INTO TABLE testtimestamphive")
+ checkAnswer(sql("select * from testtimestampcarbon"), sql("select * from testtimestamphive"))
+ sql("drop table if exists testtimestampcarbon")
+ sql("drop table if exists testtimestamphive")
+ }
+
override def afterAll {
sql("drop table if exists complexcarbontimestamptable")
sql("drop table if exists complexhivetimestamptable")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a326bb8f/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index dac4b6c..f9c3b6f 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -357,6 +357,8 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
columnsInfo.getColumnSchemaDetailsWrapper().get(eachPrimitive.getColumnId());
if (columnSchemaDetails.isDirectDictionary()) {
isDirectDictionary.add(true);
+ } else {
+ isDirectDictionary.add(false);
}
}
} else {
[21/47] incubator-carbondata git commit: [CARBONDATA-95] Columns
values with numeric data types are not getting parsed when included in
dictionary_include (#853)
Posted by ra...@apache.org.
[CARBONDATA-95] Columns values with numeric data types are not getting parsed when included in dictionary_include (#853)
Problem: Columns values with numeric data types are not getting parsed when included in dictionary_include
Analysis: When a numeric datatype lets say Decimal is defined for a column and the column is included as dictionary_include, then the whatever precision and scale is defined by the user is not taken into consideration and each value is accepted and dictionary is generated for that value.
Solution: Parse big decimal while generating global dictionary and dictionary look up and set the precision and scale specified by the user
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4b6314cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4b6314cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4b6314cc
Branch: refs/heads/master
Commit: 4b6314cc337e5af8ecffc68ee1ffd3c32f9b754e
Parents: 577764b
Author: manishgupta88 <to...@gmail.com>
Authored: Mon Jul 25 15:45:57 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 15:45:57 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/core/util/DataTypeUtil.java | 97 +++++++++++---------
.../spark/tasks/DictionaryWriterTask.scala | 48 +++++-----
.../resources/decimalBoundaryDataCarbon.csv | 22 ++---
.../test/resources/decimalBoundaryDataHive.csv | 22 ++---
.../testsuite/bigdecimal/TestBigDecimal.scala | 19 +++-
.../processing/datatypes/PrimitiveDataType.java | 29 ++++--
.../processing/mdkeygen/MDKeyGenStepMeta.java | 3 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 19 ++++
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 3 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 68 +++++++++++---
.../processing/util/CarbonSchemaParser.java | 1 +
11 files changed, 220 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
index e6d6422..d0037e1 100644
--- a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
@@ -3,15 +3,11 @@ package org.carbondata.core.util;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.carbondata.core.constants.CarbonCommonConstants;
-
-import org.apache.commons.lang.NumberUtils;
public final class DataTypeUtil {
private DataTypeUtil() {
@@ -154,76 +150,95 @@ public final class DataTypeUtil {
* @param actualDataType actual data type
* @return actual data after conversion
*/
- public static Object getDataBasedOnDataType(String data, DataType actualDataType) {
+ public static Object normalizeIntAndLongValues(String data, DataType actualDataType) {
if (null == data) {
return null;
}
try {
+ Object parsedValue = null;
switch (actualDataType) {
case INT:
+ parsedValue = Integer.parseInt(data);
+ break;
case LONG:
- case DOUBLE:
- case DECIMAL:
- if (!NumberUtils.isNumber(data)) {
- return null;
- }
- return data;
- case TIMESTAMP:
- if (data.isEmpty()) {
- return null;
- }
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(data);
- return dateToStr.getTime();
- } catch (ParseException e) {
- return null;
- }
+ parsedValue = Long.parseLong(data);
+ break;
default:
return data;
}
+ if(null != parsedValue) {
+ return data;
+ }
+ return null;
} catch (NumberFormatException ex) {
return null;
}
}
/**
- * This method will parse a given string value corresponding to its datatype
+ * This method will parse a given string value corresponding to its data type
*
- * @param value value to parse
- * @param dataType datatype for that value
+ * @param value value to parse
+ * @param dimension dimension to get data type and precision and scale in case of decimal
+ * data type
* @return
*/
- public static boolean validateColumnValueForItsDataType(String value, DataType dataType) {
+ public static String normalizeColumnValueForItsDataType(String value, CarbonDimension dimension) {
try {
Object parsedValue = null;
// validation will not be done for timestamp datatype as for timestamp direct dictionary
// is generated. No dictionary file is created for timestamp datatype column
- switch (dataType) {
+ switch (dimension.getDataType()) {
case DECIMAL:
- parsedValue = new BigDecimal(value);
- break;
+ return parseStringToBigDecimal(value, dimension);
case INT:
- parsedValue = Integer.parseInt(value);
- break;
case LONG:
- parsedValue = Long.valueOf(value);
+ parsedValue = normalizeIntAndLongValues(value, dimension.getDataType());
break;
case DOUBLE:
- parsedValue = Double.valueOf(value);
+ parsedValue = Double.parseDouble(value);
break;
default:
- return true;
+ return value;
}
if (null != parsedValue) {
- return true;
+ return value;
}
- return false;
+ return null;
} catch (Exception e) {
- return false;
+ return null;
+ }
+ }
+
+ /**
+ * This method will parse a value to its datatype if datatype is decimal else will return
+ * the value passed
+ *
+ * @param value value to be parsed
+ * @param dimension
+ * @return
+ */
+ public static String parseValue(String value, CarbonDimension dimension) {
+ try {
+ switch (dimension.getDataType()) {
+ case DECIMAL:
+ return parseStringToBigDecimal(value, dimension);
+ default:
+ return value;
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static String parseStringToBigDecimal(String value, CarbonDimension dimension) {
+ BigDecimal bigDecimal = new BigDecimal(value)
+ .setScale(dimension.getColumnSchema().getScale(), RoundingMode.HALF_UP);
+ BigDecimal normalizedValue =
+ normalizeDecimalValue(bigDecimal, dimension.getColumnSchema().getPrecision());
+ if (null != normalizedValue) {
+ return normalizedValue.toString();
}
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala b/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
index aacf402..380e76b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
@@ -65,24 +65,24 @@ class DictionaryWriterTask(valuesBuffer: mutable.HashSet[String],
if (model.dictFileExists(columnIndex)) {
if (dictionary.getSurrogateKey(values(0)) == CarbonCommonConstants
.INVALID_SURROGATE_KEY) {
- val parseSuccess = org.carbondata.core.util.DataTypeUtil
- .validateColumnValueForItsDataType(values(0),
- model.primDimensions(columnIndex).getDataType);
- if (parseSuccess) {
- writer.write(values(0))
- distinctValues.add(values(0))
+ val parsedValue = org.carbondata.core.util.DataTypeUtil
+ .normalizeColumnValueForItsDataType(values(0),
+ model.primDimensions(columnIndex))
+ if (null != parsedValue) {
+ writer.write(parsedValue)
+ distinctValues.add(parsedValue)
}
}
for (i <- 1 until values.length) {
if (preValue != values(i)) {
if (dictionary.getSurrogateKey(values(i)) ==
CarbonCommonConstants.INVALID_SURROGATE_KEY) {
- val parseSuccess = org.carbondata.core.util.DataTypeUtil
- .validateColumnValueForItsDataType(values(i),
- model.primDimensions(columnIndex).getDataType);
- if (parseSuccess) {
- writer.write(values(i))
- distinctValues.add(values(i))
+ val parsedValue = org.carbondata.core.util.DataTypeUtil
+ .normalizeColumnValueForItsDataType(values(i),
+ model.primDimensions(columnIndex))
+ if (null != parsedValue) {
+ writer.write(parsedValue)
+ distinctValues.add(parsedValue)
preValue = values(i)
}
}
@@ -90,21 +90,21 @@ class DictionaryWriterTask(valuesBuffer: mutable.HashSet[String],
}
} else {
- val parseSuccess = org.carbondata.core.util.DataTypeUtil
- .validateColumnValueForItsDataType(values(0),
- model.primDimensions(columnIndex).getDataType);
- if (parseSuccess) {
- writer.write(values(0))
- distinctValues.add(values(0))
+ val parsedValue = org.carbondata.core.util.DataTypeUtil
+ .normalizeColumnValueForItsDataType(values(0),
+ model.primDimensions(columnIndex))
+ if (null != parsedValue) {
+ writer.write(parsedValue)
+ distinctValues.add(parsedValue)
}
for (i <- 1 until values.length) {
if (preValue != values(i)) {
- val parseSuccess = org.carbondata.core.util.DataTypeUtil
- .validateColumnValueForItsDataType(values(i),
- model.primDimensions(columnIndex).getDataType);
- if (parseSuccess) {
- writer.write(values(i))
- distinctValues.add(values(i))
+ val parsedValue = org.carbondata.core.util.DataTypeUtil
+ .normalizeColumnValueForItsDataType(values(i),
+ model.primDimensions(columnIndex))
+ if (null != parsedValue) {
+ writer.write(parsedValue)
+ distinctValues.add(parsedValue)
preValue = values(i)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv b/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
index c64a9bf..1744362 100644
--- a/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
+++ b/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
@@ -1,12 +1,12 @@
ID,date,country,name,phonetype,serialname,salary
-1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.0000000000
-2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.0000000000
-3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.0000000000
-4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.0000000000
-5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.0000000000
-6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.0000000000
-7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.0000000000
-8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.0000000000
-9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.0000000000
-10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.0000000000
-11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.0000000000
+1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.1234567890123
+2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.1234567890123
+3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.1234567890123
+4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.1234567890123
+5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.1234567890123
+6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.1234567890123
+7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.1234567890123
+8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.1234567890123
+9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.1234567890123
+10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.1234567890123
+11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.1234567890123
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/decimalBoundaryDataHive.csv b/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
index a2faaf1..2bcf7f5 100644
--- a/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
+++ b/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
@@ -1,11 +1,11 @@
-1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.0000000000
-2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.0000000000
-3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.0000000000
-4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.0000000000
-5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.0000000000
-6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.0000000000
-7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.0000000000
-8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.0000000000
-9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.0000000000
-10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.0000000000
-11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.0000000000
+1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.1234567890123
+2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.1234567890123
+3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.1234567890123
+4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.1234567890123
+5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.1234567890123
+6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.1234567890123
+7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.1234567890123
+8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.1234567890123
+9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.1234567890123
+10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.1234567890123
+11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.1234567890123
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index 95dd1b4..b07705d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -34,12 +34,15 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("drop table if exists carbonTable")
sql("drop table if exists hiveTable")
+ sql("drop table if exists hiveBigDecimal")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql("CREATE TABLE IF NOT EXISTS carbonTable (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary Decimal(17,2))STORED BY 'org.apache.carbondata.format'")
sql("create table if not exists hiveTable(ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary Decimal(17,2))row format delimited fields terminated by ','")
sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalDataWithHeader.csv' into table carbonTable")
sql("LOAD DATA local inpath './src/test/resources/decimalDataWithoutHeader.csv' INTO table hiveTable")
+ sql("create table if not exists hiveBigDecimal(ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10))row format delimited fields terminated by ','")
+ sql("LOAD DATA local inpath './src/test/resources/decimalBoundaryDataHive.csv' INTO table hiveBigDecimal")
}
test("test detail query on big decimal column") {
@@ -111,11 +114,8 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
test("test aggregation on big decimal column with increased precision") {
sql("drop table if exists carbonBigDecimal")
- sql("drop table if exists hiveBigDecimal")
sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10)) STORED BY 'org.apache.carbondata.format'")
- sql("create table if not exists hiveBigDecimal(ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10))row format delimited fields terminated by ','")
sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
- sql("LOAD DATA local inpath './src/test/resources/decimalBoundaryDataHive.csv' INTO table hiveBigDecimal")
checkAnswer(sql("select sum(salary) from carbonBigDecimal"),
sql("select sum(salary) from hiveBigDecimal"))
@@ -124,13 +124,24 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql("select sum(distinct salary) from hiveBigDecimal"))
sql("drop table if exists carbonBigDecimal")
- sql("drop table if exists hiveBigDecimal")
+ }
+
+ test("test big decimal for dictionary look up") {
+ sql("drop table if exists decimalDictLookUp")
+ sql("create table if not exists decimalDictLookUp (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10)) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='salary')")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table decimalDictLookUp")
+
+ checkAnswer(sql("select sum(salary) from decimalDictLookUp"),
+ sql("select sum(salary) from hiveBigDecimal"))
+
+ sql("drop table if exists decimalDictLookUp")
}
override def afterAll {
sql("drop table if exists carbonTable")
sql("drop table if exists hiveTable")
+ sql("drop table if exists hiveBigDecimal")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
index e63d727..3d272d5 100644
--- a/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.keygenerator.KeyGenException;
import org.carbondata.core.keygenerator.KeyGenerator;
+import org.carbondata.core.util.DataTypeUtil;
import org.carbondata.processing.surrogatekeysgenerator.csvbased.CarbonCSVBasedDimSurrogateKeyGen;
import org.pentaho.di.core.exception.KettleException;
@@ -58,6 +59,11 @@ public class PrimitiveDataType implements GenericDataType {
private String columnId;
/**
+ * dimension ordinal of primitive type column
+ */
+ private int dimensionOrdinal;
+
+ /**
* key size
*/
private int keySize;
@@ -74,14 +80,16 @@ public class PrimitiveDataType implements GenericDataType {
/**
* constructor
+ *
* @param name
* @param parentname
* @param columnId
*/
- public PrimitiveDataType(String name, String parentname, String columnId) {
+ public PrimitiveDataType(String name, String parentname, String columnId, int dimensionOrdinal) {
this.name = name;
this.parentname = parentname;
this.columnId = columnId;
+ this.dimensionOrdinal = dimensionOrdinal;
}
/*
@@ -151,15 +159,22 @@ public class PrimitiveDataType implements GenericDataType {
/*
* parse string and generate surrogate
*/
- @Override
- public void parseStringAndWriteByteArray(String tableName, String inputString,
+ @Override public void parseStringAndWriteByteArray(String tableName, String inputString,
String[] delimiter, int delimiterIndex, DataOutputStream dataOutputStream,
CarbonCSVBasedDimSurrogateKeyGen surrogateKeyGen) throws KettleException, IOException {
- Integer surrogateKey = surrogateKeyGen
- .generateSurrogateKeys(inputString, tableName + CarbonCommonConstants.UNDERSCORE + name,
- this.getColumnId());
- if (surrogateKey == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
+ String parsedValue = DataTypeUtil.parseValue(inputString,
+ surrogateKeyGen.getDimensionNameToDimensionMapping()
+ .get(tableName + CarbonCommonConstants.UNDERSCORE + name));
+ Integer surrogateKey = null;
+ if (null == parsedValue) {
surrogateKey = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
+ } else {
+ surrogateKey = surrogateKeyGen
+ .generateSurrogateKeys(parsedValue, tableName + CarbonCommonConstants.UNDERSCORE + name,
+ this.getColumnId());
+ if (surrogateKey == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
+ surrogateKey = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
+ }
}
dataOutputStream.writeInt(surrogateKey);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java b/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
index 9b65e63..902de1e 100644
--- a/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
+++ b/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
@@ -496,7 +496,8 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
g.addChildren(new StructDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
break;
default:
- g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
+ g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3],
+ Integer.parseInt(levelInfo[4])));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index f9c3b6f..8fc1196 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -33,6 +33,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.cache.dictionary.Dictionary;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.file.manager.composite.IFileManagerComposite;
import org.carbondata.core.keygenerator.KeyGenException;
@@ -116,6 +117,10 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
new HashMap<String, Map<ArrayWrapper, Integer>>(
CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
/**
+ * dimension name to dimension mapping
+ */
+ private Map<String, CarbonDimension> dimensionNameToDimensionMapping;
+ /**
* rwLock2
*/
private ReentrantReadWriteLock rwLock2 = new ReentrantReadWriteLock();
@@ -512,4 +517,18 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
this.measureMaxSurroagetMap = measureMaxSurroagetMap;
}
+ /**
+ * @return
+ */
+ public Map<String, CarbonDimension> getDimensionNameToDimensionMapping() {
+ return dimensionNameToDimensionMapping;
+ }
+
+ /**
+ * @param dimensionNameToDimensionMapping
+ */
+ public void setDimensionNameToDimensionMapping(
+ Map<String, CarbonDimension> dimensionNameToDimensionMapping) {
+ this.dimensionNameToDimensionMapping = dimensionNameToDimensionMapping;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
index dad2bbc..e96be83 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
@@ -1026,7 +1026,8 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
g.addChildren(new StructDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
break;
default:
- g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3]));
+ g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3],
+ Integer.parseInt(levelInfo[4])));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 2978ad2..207e68d 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -48,6 +48,7 @@ import org.carbondata.core.cache.dictionary.Dictionary;
import org.carbondata.core.carbon.metadata.CarbonMetadata;
import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.carbon.metadata.schema.table.CarbonTable;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.file.manager.composite.FileData;
@@ -57,7 +58,10 @@ import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
-import org.carbondata.core.util.*;
+import org.carbondata.core.util.CarbonProperties;
+import org.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.util.DataTypeUtil;
import org.carbondata.core.writer.ByteArrayHolder;
import org.carbondata.core.writer.HierarchyValueWriterForCSV;
import org.carbondata.processing.dataprocessor.manager.CarbonDataProcessorManager;
@@ -374,7 +378,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
surrogateKeyGen = new FileStoreSurrogateKeyGenForCSV(columnsInfo, meta.getPartitionID(),
meta.getSegmentId(), meta.getTaskNo());
data.setSurrogateKeyGen(surrogateKeyGen);
-
updateStoreLocation();
// Check the insert hierarchies required or not based on that
@@ -431,6 +434,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
columnSchemaDetailsWrapper = meta.getColumnSchemaDetailsWrapper();
if (null != getInputRowMeta()) {
generateNoDictionaryAndComplexIndexMapping();
+ data.getSurrogateKeyGen()
+ .setDimensionNameToDimensionMapping(populateNameToCarbonDimensionMap());
}
serializationNullFormat = meta.getTableOptionWrapper().get("serialization_null_format");
}
@@ -933,8 +938,9 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
String columnName = metaColumnNames[j];
String foreignKeyColumnName = foreignKeyMappingColumns[j];
// check if it is ignore dictionary dimension or not . if yes directly write byte buffer
+ String tuple = (String) r[j];
if (isNoDictionaryColumn[j]) {
- processnoDictionaryDim(noDictionaryAndComplexIndexMapping[j], (String) r[j], dataTypes[j],
+ processnoDictionaryDim(noDictionaryAndComplexIndexMapping[j], tuple, dataTypes[j],
isStringDataType[j], byteBufferArr);
continue;
}
@@ -1028,7 +1034,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
int[] surrogateKeyForHierarchy = null;
if (null != cache) {
- Integer keyFromCsv = dicCache.getSurrogateKey(((String) r[j]));
+ Integer keyFromCsv = dicCache.getSurrogateKey(tuple);
if (null != keyFromCsv) {
surrogateKeyForHierarchy = cache.get(keyFromCsv);
@@ -1049,7 +1055,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
} else {
surrogateKeyForHierarchy = new int[1];
surrogateKeyForHierarchy[0] =
- surrogateKeyGen.generateSurrogateKeys((String) r[j], foreignKeyColumnName);
+ surrogateKeyGen.generateSurrogateKeys(tuple, foreignKeyColumnName);
}
for (int k = 0; k < surrogateKeyForHierarchy.length; k++) {
if (dimPresentCsvOrder[i]) {
@@ -1069,7 +1075,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
complexTypes[noDictionaryAndComplexIndexMapping[j] - meta.noDictionaryCols.length];
ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteArray);
- complexType.parseStringAndWriteByteArray(meta.getTableName(), (String) r[j],
+ complexType.parseStringAndWriteByteArray(meta.getTableName(), tuple,
new String[] { meta.getComplexDelimiterLevel1(), meta.getComplexDelimiterLevel2() },
0, dataOutputStream, surrogateKeyGen);
byteBufferArr[noDictionaryAndComplexIndexMapping[j]] =
@@ -1094,7 +1100,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
Int2ObjectMap<int[]> cache = surrogateKeyGen.getHierCache().get(actualHierName);
int[] surrogateKeyForHrrchy = null;
if (null != cache) {
- Integer keyFromCsv = dicCache.getSurrogateKey(((String) r[j]));
+ Integer keyFromCsv = dicCache.getSurrogateKey(tuple);
if (null != keyFromCsv) {
surrogateKeyForHrrchy = cache.get(keyFromCsv);
@@ -1137,12 +1143,20 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(details.getColumnType());
surrogateKeyForHrrchy[0] =
- directDictionaryGenerator1.generateDirectSurrogateKey(((String) r[j]));
+ directDictionaryGenerator1.generateDirectSurrogateKey(tuple);
surrogateKeyGen.max[m] = Integer.MAX_VALUE;
} else {
- surrogateKeyForHrrchy[0] =
- surrogateKeyGen.generateSurrogateKeys(((String) r[j]), foreignKeyColumnName);
+ String parsedValue = DataTypeUtil.parseValue(tuple,
+ data.getSurrogateKeyGen().getDimensionNameToDimensionMapping().get(
+ meta.getTableName() + CarbonCommonConstants.UNDERSCORE + columnName
+ .toLowerCase()));
+ if(null == parsedValue) {
+ surrogateKeyForHrrchy[0] = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
+ } else {
+ surrogateKeyForHrrchy[0] =
+ surrogateKeyGen.generateSurrogateKeys(parsedValue, foreignKeyColumnName);
+ }
}
}
if (surrogateKeyForHrrchy[0] == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
@@ -1763,7 +1777,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
boolean isStringDataType, ByteBuffer[] out) {
if (!(isStringDataType)) {
if (null == DataTypeUtil
- .getDataBasedOnDataType(dimensionValue, DataTypeUtil.getDataType(dataType))) {
+ .normalizeIntAndLongValues(dimensionValue, DataTypeUtil.getDataType(dataType))) {
dimensionValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
}
}
@@ -1824,5 +1838,37 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
}
}
+ private Map<String, CarbonDimension> populateNameToCarbonDimensionMap() {
+ CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
+ meta.getSchemaName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
+ List<CarbonDimension> dimensionsList = carbonTable.getDimensionByTableName(meta.getTableName());
+ Map<String, CarbonDimension> dimensionNameToDimensionMapping =
+ new HashMap<>(dimensionsList.size());
+ for (CarbonDimension dimension : dimensionsList) {
+ if (dimension.isComplex()) {
+ populateComplexDimension(dimensionNameToDimensionMapping, dimension);
+ } else {
+ dimensionNameToDimensionMapping.put(
+ meta.getTableName() + CarbonCommonConstants.UNDERSCORE + dimension.getColName()
+ .toLowerCase(), dimension);
+ }
+ }
+ return dimensionNameToDimensionMapping;
+ }
+
+ private void populateComplexDimension(
+ Map<String, CarbonDimension> dimensionNameToDimensionMapping, CarbonDimension dimension) {
+ List<CarbonDimension> listOfChildDimensions = dimension.getListOfChildDimensions();
+ for (CarbonDimension childDimension : listOfChildDimensions) {
+ if (childDimension.isComplex()) {
+ populateComplexDimension(dimensionNameToDimensionMapping, childDimension);
+ } else {
+ dimensionNameToDimensionMapping.put(
+ meta.getTableName() + CarbonCommonConstants.UNDERSCORE + childDimension.getColName(),
+ childDimension);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4b6314cc/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java b/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java
index e0b1f1a..119e307 100644
--- a/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java
+++ b/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java
@@ -1179,6 +1179,7 @@ public final class CarbonSchemaParser {
childDim.getColName() + CarbonCommonConstants.COLON_SPC_CHARACTER + childDim
.getDataType() + CarbonCommonConstants.COLON_SPC_CHARACTER + dimension.getColName()
+ CarbonCommonConstants.COLON_SPC_CHARACTER + childDim.getColumnId()
+ + CarbonCommonConstants.COLON_SPC_CHARACTER + childDim.getOrdinal()
+ CarbonCommonConstants.HASH_SPC_CHARACTER);
}
}
[04/47] incubator-carbondata git commit: [CARBONDATA-17] - select
count(*) from table where column_x = 'value' is not returning the correct
count (#771)
Posted by ra...@apache.org.
[CARBONDATA-17] - select count(*) from table where column_x = 'value' is not returning the correct count (#771)
if the first measure is having all values null.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/34741b48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/34741b48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/34741b48
Branch: refs/heads/master
Commit: 34741b48681b1a50bb46d84615dd77c17a6c44f7
Parents: 6b7b41a
Author: Mohammad Shahid Khan <mo...@gmail.com>
Authored: Fri Jul 22 15:12:17 2016 +0530
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Fri Jul 22 15:12:17 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 ++
.../query/aggregator/impl/CountAggregator.java | 2 +-
.../aggregator/impl/CountStarAggregator.java | 51 ++++++++++++++
.../util/MeasureAggregatorFactory.java | 28 ++------
.../org/apache/spark/sql/CarbonOperators.scala | 4 +-
.../src/test/resources/datanullmeasurecol.csv | 3 +
.../filterexpr/CountStarTestCase.scala | 72 ++++++++++++++++++++
7 files changed, 137 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 8dbe4dd..a7a8c07 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -155,6 +155,10 @@ public final class CarbonCommonConstants {
*/
public static final String COUNT = "count";
/**
+ * Count constant
+ */
+ public static final String COUNT_STAR = "countstar";
+ /**
* Max constant
*/
public static final String MAX = "max";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java b/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
index 12c2061..1a040b6 100644
--- a/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
+++ b/core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
@@ -42,7 +42,7 @@ public class CountAggregator implements MeasureAggregator {
/**
* aggregate value
*/
- private double aggVal;
+ protected double aggVal;
/**
* Count Aggregate function which update the total count
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java b/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
new file mode 100644
index 0000000..3b7e78c
--- /dev/null
+++ b/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.query.aggregator.impl;
+
+import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.carbondata.query.aggregator.MeasureAggregator;
+
+/**
+ * Class Description : It will return total count of values
+ */
+public class CountStarAggregator extends CountAggregator {
+
+ @Override public void agg(double newVal) {
+ super.agg(newVal);
+ }
+
+ @Override public void agg(Object newVal) {
+ aggVal++;
+ }
+
+ @Override public void agg(MeasureColumnDataChunk dataChunk, int index) {
+ aggVal++;
+ }
+
+ @Override public MeasureAggregator getCopy() {
+ CountStarAggregator aggregator = new CountStarAggregator();
+ aggregator.aggVal = aggVal;
+ return aggregator;
+ }
+
+ @Override public MeasureAggregator getNew() {
+ return new CountStarAggregator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java b/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
index a49c374..3fbf819 100644
--- a/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
+++ b/core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
@@ -24,30 +24,7 @@ import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.query.aggregator.CustomMeasureAggregator;
import org.carbondata.query.aggregator.MeasureAggregator;
-import org.carbondata.query.aggregator.impl.AvgBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.AvgDoubleAggregator;
-import org.carbondata.query.aggregator.impl.AvgLongAggregator;
-import org.carbondata.query.aggregator.impl.AvgTimestampAggregator;
-import org.carbondata.query.aggregator.impl.CountAggregator;
-import org.carbondata.query.aggregator.impl.DistinctCountAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DistinctCountBigDecimalAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DistinctCountLongAggregatorObjectSet;
-import org.carbondata.query.aggregator.impl.DummyBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.DummyDoubleAggregator;
-import org.carbondata.query.aggregator.impl.DummyLongAggregator;
-import org.carbondata.query.aggregator.impl.MaxAggregator;
-import org.carbondata.query.aggregator.impl.MaxBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.MaxLongAggregator;
-import org.carbondata.query.aggregator.impl.MinAggregator;
-import org.carbondata.query.aggregator.impl.MinBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.MinLongAggregator;
-import org.carbondata.query.aggregator.impl.SumBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctBigDecimalAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctDoubleAggregator;
-import org.carbondata.query.aggregator.impl.SumDistinctLongAggregator;
-import org.carbondata.query.aggregator.impl.SumDoubleAggregator;
-import org.carbondata.query.aggregator.impl.SumLongAggregator;
-import org.carbondata.query.aggregator.impl.SumTimestampAggregator;
+import org.carbondata.query.aggregator.impl.*;
import org.carbondata.query.carbon.model.CustomAggregateExpression;
/**
@@ -106,6 +83,9 @@ public class MeasureAggregatorFactory {
} else if (CarbonCommonConstants.COUNT.equalsIgnoreCase(aggregatorType)) {
return new CountAggregator();
}
+ else if (CarbonCommonConstants.COUNT_STAR.equalsIgnoreCase(aggregatorType)) {
+ return new CountStarAggregator();
+ }
//
else if (CarbonCommonConstants.MAX.equalsIgnoreCase(aggregatorType)) {
switch (dataType) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
index 9946dfd..76b7e8c 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
@@ -113,7 +113,7 @@ case class CarbonTableScan(
case CountCarbon(posLiteral@PositionLiteral(Literal(star, _), _)) if star == "*" =>
val m1 = new QueryMeasure("count(*)")
- m1.setAggregateFunction(CarbonCommonConstants.COUNT)
+ m1.setAggregateFunction(CarbonCommonConstants.COUNT_STAR)
m1.setQueryOrder(queryOrder)
posLiteral.setPosition(queryOrder)
plan.addMeasure(m1)
@@ -123,7 +123,7 @@ case class CarbonTableScan(
case curr@CountCarbon(posLiteral@PositionLiteral(one, _)) =>
val m1 = new QueryMeasure("count(*)")
- m1.setAggregateFunction(CarbonCommonConstants.COUNT)
+ m1.setAggregateFunction(CarbonCommonConstants.COUNT_STAR)
m1.setQueryOrder(queryOrder)
posLiteral.setPosition(queryOrder)
plan.addMeasure(m1)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/test/resources/datanullmeasurecol.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datanullmeasurecol.csv b/integration/spark/src/test/resources/datanullmeasurecol.csv
new file mode 100644
index 0000000..40692d8
--- /dev/null
+++ b/integration/spark/src/test/resources/datanullmeasurecol.csv
@@ -0,0 +1,3 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,A234
+2,2015/7/24,china,aaa2,phone756,A453
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/34741b48/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
new file mode 100644
index 0000000..20881c5
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/CountStarTestCase.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.spark.testsuite.filterexpr
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.util.CarbonProperties
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for filter expression query on String datatypes
+ *
+ * @author N00902756
+ *
+ */
+class CountStarTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("drop table if exists filtertestTables")
+
+
+ sql("CREATE TABLE filterTimestampDataType (ID int, date Timestamp, country String, " +
+ "name String, phonetype String, serialname String, salary int) " +
+ "STORED BY 'org.apache.carbondata.format'"
+ )
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ var csvFilePath = currentDirectory + "/src/test/resources/datanullmeasurecol.csv";
+ sql(
+ s"LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE " +
+ s"filterTimestampDataType " +
+ s"OPTIONS('DELIMITER'= ',', " +
+ s"'FILEHEADER'= '')"
+ )
+ }
+
+ test("select count ") {
+ checkAnswer(
+ sql("select count(*) from filterTimestampDataType where country='china'"),
+ Seq(Row(2))
+ )
+ }
+
+ override def afterAll {
+ sql("drop table noloadtable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+}
\ No newline at end of file
[44/47] incubator-carbondata git commit: Merge remote-tracking branch
'carbon_master/master' into apache/master
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
index 056a936,0000000..23244f8
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
@@@ -1,1332 -1,0 +1,1364 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.scan.filter;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.cache.Cache;
+import org.carbondata.core.cache.CacheProvider;
+import org.carbondata.core.cache.CacheType;
+import org.carbondata.core.cache.dictionary.Dictionary;
+import org.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
+import org.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.carbondata.core.cache.dictionary.ForwardDictionary;
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.datastore.IndexKey;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.keygenerator.KeyGenerator;
+import org.carbondata.core.util.ByteUtil;
+import org.carbondata.core.util.CarbonProperties;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.util.CarbonUtilException;
+import org.carbondata.core.util.DataTypeUtil;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.ColumnExpression;
+import org.carbondata.scan.expression.Expression;
+import org.carbondata.scan.expression.ExpressionResult;
+import org.carbondata.scan.expression.LiteralExpression;
++import org.carbondata.scan.expression.UnknownExpression;
+import org.carbondata.scan.expression.conditional.ListExpression;
+import org.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.carbondata.scan.filter.executer.*;
+import org.carbondata.scan.filter.intf.ExpressionType;
+import org.carbondata.scan.filter.intf.FilterExecuterType;
+import org.carbondata.scan.filter.intf.RowImpl;
+import org.carbondata.scan.filter.intf.RowIntf;
+import org.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.carbondata.scan.filter.resolver.RowLevelFilterResolverImpl;
+import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public final class FilterUtil {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(FilterUtil.class.getName());
+
+ private FilterUtil() {
+
+ }
+
+ /**
+ * Pattern used : Visitor Pattern
+ * Method will create filter executer tree based on the filter resolved tree,
+ * in this algorithm based on the resolver instance the executers will be visited
+ * and the resolved surrogates will be converted to keys
+ *
+ * @param filterExpressionResolverTree
+ * @param segmentProperties
+ * @return FilterExecuter instance
+ */
+ private static FilterExecuter createFilterExecuterTree(
+ FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
+ Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ FilterExecuterType filterExecuterType = filterExpressionResolverTree.getFilterExecuterType();
+ if (null != filterExecuterType) {
+ switch (filterExecuterType) {
+ case INCLUDE:
+ return getIncludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
+ case EXCLUDE:
+ return getExcludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
+ case OR:
+ return new OrFilterExecuterImpl(
+ createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
+ complexDimensionInfoMap),
+ createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
+ complexDimensionInfoMap));
+ case AND:
+ return new AndFilterExecuterImpl(
+ createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
+ complexDimensionInfoMap),
+ createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
+ complexDimensionInfoMap));
+ case RESTRUCTURE:
+ return new RestructureFilterExecuterImpl(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(),
+ segmentProperties.getDimensionKeyGenerator());
+ case ROWLEVEL_LESSTHAN:
+ case ROWLEVEL_LESSTHAN_EQUALTO:
+ case ROWLEVEL_GREATERTHAN_EQUALTO:
+ case ROWLEVEL_GREATERTHAN:
+ return RowLevelRangeTypeExecuterFacory
+ .getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree,
+ segmentProperties);
+ case ROWLEVEL:
+ default:
+ return new RowLevelFilterExecuterImpl(
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties, complexDimensionInfoMap);
+
+ }
+ }
+ return new RowLevelFilterExecuterImpl(
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getDimColEvaluatorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getMsrColEvalutorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties, complexDimensionInfoMap);
+
+ }
+
+ /**
+ * It gives filter executer based on columnar or column group
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return
+ */
+ private static FilterExecuter getIncludeFilterExecuter(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
+
+ if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+ return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ } else {
+ return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ }
+ }
+
+ /**
+ * It gives filter executer based on columnar or column group
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return
+ */
+ private static FilterExecuter getExcludeFilterExecuter(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
+
+ if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+ return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ } else {
+ return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ }
+ }
+
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfExpressionContainsColumn(Expression expression) {
+ if (expression instanceof ColumnExpression) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfExpressionContainsColumn(child)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfLeftExpressionRequireEvaluation(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
+ || !(expression instanceof ColumnExpression)) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfLeftExpressionRequireEvaluation(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This method will check if a given literal expression is not a timestamp datatype
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfDataTypeNotTimeStamp(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.LITERAL) {
+ if (!(((LiteralExpression) expression).getLiteralExpDataType()
+ == DataType.TIMESTAMP)) {
+ return true;
+ }
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfDataTypeNotTimeStamp(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfRightExpressionRequireEvaluation(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
+ || !(expression instanceof LiteralExpression) && !(expression instanceof ListExpression)) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfRightExpressionRequireEvaluation(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * method will get the masked keys based on the keys generated from surrogates.
+ *
+ * @param ranges
+ * @param key
+ * @return byte[]
+ */
+ private static byte[] getMaskedKey(int[] ranges, byte[] key) {
+ byte[] maskkey = new byte[ranges.length];
+
+ for (int i = 0; i < maskkey.length; i++) {
+ maskkey[i] = key[ranges[i]];
+ }
+ return maskkey;
+ }
+
+ /**
+ * This method will return the ranges for the masked Bytes based on the key
+ * Generator.
+ *
+ * @param queryDimensionsOrdinal
+ * @param generator
+ * @return
+ */
+ private static int[] getRangesForMaskedByte(int queryDimensionsOrdinal, KeyGenerator generator) {
+ Set<Integer> integers = new TreeSet<Integer>();
+ int[] range = generator.getKeyByteOffsets(queryDimensionsOrdinal);
+ for (int j = range[0]; j <= range[1]; j++) {
+ integers.add(j);
+ }
+
+ int[] byteIndexs = new int[integers.size()];
+ int j = 0;
+ for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
+ Integer integer = iterator.next();
+ byteIndexs[j++] = integer.intValue();
+ }
+ return byteIndexs;
+ }
+
+ /**
+ * This method will get the no dictionary data based on filters and same
+ * will be in DimColumnFilterInfo
+ *
+ * @param tableIdentifier
+ * @param columnExpression
+ * @param evaluateResultListFinal
+ * @param isIncludeFilter
+ * @return DimColumnFilterInfo
+ */
+ public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
+ AbsoluteTableIdentifier tableIdentifier, ColumnExpression columnExpression,
+ List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ for (String result : evaluateResultListFinal) {
+ filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ }
+
+ Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
+
+ @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
+ // TODO Auto-generated method stub
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
+ }
+
+ };
+ Collections.sort(filterValuesList, filterNoDictValueComaparator);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (filterValuesList.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterListForNoDictionaryCols(filterValuesList);
+
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates.
+ *
+ * @param tableIdentifier
+ * @param columnExpression
+ * @param evaluateResultList
+ * @param isIncludeFilter
+ * @return
+ * @throws QueryExecutionException
+ */
+ public static DimColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier,
+ ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter)
+ throws QueryExecutionException {
+ Dictionary forwardDictionary = null;
+ try {
+ // Reading the dictionary value from cache.
+ forwardDictionary =
+ getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
+ return getFilterValues(columnExpression, evaluateResultList, forwardDictionary,
+ isIncludeFilter);
+ } finally {
+ CarbonUtil.clearDictionaryCache(forwardDictionary);
+ }
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates.
+ *
+ * @param columnExpression
+ * @param evaluateResultList
+ * @param forwardDictionary
+ * @param isIncludeFilter
+ * @return
+ * @throws QueryExecutionException
+ */
+ private static DimColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
+ List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter)
+ throws QueryExecutionException {
+ sortFilterModelMembers(columnExpression, evaluateResultList);
+ List<Integer> surrogates =
+ new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // Reading the dictionary value from cache.
+ getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
+ Collections.sort(surrogates);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (surrogates.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterList(surrogates);
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * This API will get the Dictionary value for the respective filter member
+ * string.
+ *
+ * @param evaluateResultList filter value
+ * @param surrogates
+ * @throws QueryExecutionException
+ */
+ private static void getDictionaryValue(List<String> evaluateResultList,
+ Dictionary forwardDictionary, List<Integer> surrogates) throws QueryExecutionException {
+ ((ForwardDictionary) forwardDictionary)
+ .getSurrogateKeyByIncrementalSearch(evaluateResultList, surrogates);
+ }
+
+ /**
+ * This method will get all the members of column from the forward dictionary
+ * cache, this method will be basically used in row level filter resolver.
+ *
+ * @param tableIdentifier
+ * @param expression
+ * @param columnExpression
+ * @param isIncludeFilter
+ * @return DimColumnFilterInfo
+ * @throws FilterUnsupportedException
+ * @throws QueryExecutionException
+ */
+ public static DimColumnFilterInfo getFilterListForAllValues(
+ AbsoluteTableIdentifier tableIdentifier, Expression expression,
+ final ColumnExpression columnExpression, boolean isIncludeFilter)
+ throws FilterUnsupportedException {
+ Dictionary forwardDictionary = null;
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ DictionaryChunksWrapper dictionaryWrapper = null;
+ try {
+ forwardDictionary =
+ getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
+ dictionaryWrapper = forwardDictionary.getDictionaryChunks();
+ while (dictionaryWrapper.hasNext()) {
+ byte[] columnVal = dictionaryWrapper.next();
+ try {
+ RowIntf row = new RowImpl();
+ String stringValue =
+ new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ stringValue = null;
+ }
+ row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt && !(rslt ^ isIncludeFilter)) {
+ if (null == stringValue) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } else {
+ evaluateResultListFinal.add(stringValue);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.debug(e.getMessage());
+ }
+ }
+ return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
+ isIncludeFilter);
+ } catch (QueryExecutionException e) {
+ throw new FilterUnsupportedException(e.getMessage());
+ } finally {
+ CarbonUtil.clearDictionaryCache(forwardDictionary);
+ }
+ }
+
+ private static void sortFilterModelMembers(final ColumnExpression columnExpression,
+ List<String> evaluateResultListFinal) {
+ Comparator<String> filterActualValueComaparator = new Comparator<String>() {
+
+ @Override public int compare(String filterMember1, String filterMember2) {
+ return compareFilterMembersBasedOnActualDataType(filterMember1, filterMember2,
+ columnExpression.getDataType());
+ }
+
+ };
+ Collections.sort(evaluateResultListFinal, filterActualValueComaparator);
+ }
+
+ /**
+ * Metahod will resolve the filter member to its respective surrogates by
+ * scanning the dictionary cache.
+ *
+ * @param tableIdentifier
+ * @param expression
+ * @param columnExpression
+ * @param isIncludeFilter
+ * @return
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ public static DimColumnFilterInfo getFilterList(AbsoluteTableIdentifier tableIdentifier,
+ Expression expression, ColumnExpression columnExpression, boolean isIncludeFilter)
+ throws QueryExecutionException, FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ try {
+ List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
+ for (ExpressionResult result : evaluateResultList) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
+ }
+
+ if (null != columnExpression.getCarbonColumn() && !columnExpression.getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY)) {
+ resolvedFilterObject =
+ getNoDictionaryValKeyMemberForFilter(tableIdentifier, columnExpression,
+ evaluateResultListFinal, isIncludeFilter);
+ } else {
+ resolvedFilterObject =
+ getFilterValues(tableIdentifier, columnExpression, evaluateResultListFinal,
+ isIncludeFilter);
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.audit(e.getMessage());
+ }
+ return resolvedFilterObject;
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates in the scenario of restructure.
+ *
+ * @param expression
+ * @param columnExpression
+ * @param defaultValues
+ * @param defaultSurrogate
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ public static DimColumnFilterInfo getFilterListForRS(Expression expression,
+ ColumnExpression columnExpression, String defaultValues, int defaultSurrogate)
+ throws FilterUnsupportedException {
+ List<Integer> filterValuesList = new ArrayList<Integer>(20);
+ DimColumnFilterInfo columnFilterInfo = null;
+ // List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ // KeyGenerator keyGenerator =
+ // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
+ try {
+ List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
+ for (ExpressionResult result : evaluateResultList) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
+ }
+
+ for (int i = 0; i < evaluateResultListFinal.size(); i++) {
+ if (evaluateResultListFinal.get(i).equals(defaultValues)) {
+ filterValuesList.add(defaultSurrogate);
+ break;
+ }
+ }
+ if (filterValuesList.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setFilterList(filterValuesList);
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.audit(e.getMessage());
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * This method will get the member based on filter expression evaluation from the
+ * forward dictionary cache, this method will be basically used in restructure.
+ *
+ * @param expression
+ * @param columnExpression
+ * @param defaultValues
+ * @param defaultSurrogate
+ * @param isIncludeFilter
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ public static DimColumnFilterInfo getFilterListForAllMembersRS(Expression expression,
+ ColumnExpression columnExpression, String defaultValues, int defaultSurrogate,
+ boolean isIncludeFilter) throws FilterUnsupportedException {
+ List<Integer> filterValuesList = new ArrayList<Integer>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ DimColumnFilterInfo columnFilterInfo = null;
+
+ // KeyGenerator keyGenerator =
+ // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
+ try {
+ RowIntf row = new RowImpl();
+ if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ defaultValues = null;
+ }
+ row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(defaultValues,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt && !(rslt ^ isIncludeFilter)) {
+ if (null == defaultValues) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } else {
+ evaluateResultListFinal.add(defaultValues);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.audit(e.getMessage());
+ }
+
+ if (null == defaultValues) {
+ defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ }
+ columnFilterInfo = new DimColumnFilterInfo();
+ for (int i = 0; i < evaluateResultListFinal.size(); i++) {
+ if (evaluateResultListFinal.get(i).equals(defaultValues)) {
+ filterValuesList.add(defaultSurrogate);
+ break;
+ }
+ }
+ columnFilterInfo.setFilterList(filterValuesList);
+ return columnFilterInfo;
+ }
+
+ public static byte[][] getKeyArray(DimColumnFilterInfo dimColumnFilterInfo,
+ CarbonDimension carbonDimension, KeyGenerator blockLevelKeyGenerator) {
+ if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
+ return dimColumnFilterInfo.getNoDictionaryFilterValuesList()
+ .toArray((new byte[dimColumnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
+ }
+ int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
+ List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ Arrays.fill(keys, 0);
+ int[] rangesForMaskedByte =
+ getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+ if (null != dimColumnFilterInfo) {
+ for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
+ try {
+ keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ filterValuesList
+ .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+
+ }
+ return filterValuesList.toArray(new byte[filterValuesList.size()][]);
+
+ }
+
+ /**
+ * The method is used to get the single dictionary key's mask key
+ *
+ * @param surrogate
+ * @param carbonDimension
+ * @param blockLevelKeyGenerator
+ * @return
+ */
+ public static byte[] getMaskKey(int surrogate, CarbonDimension carbonDimension,
+ KeyGenerator blockLevelKeyGenerator) {
+
+ int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
+ byte[] maskedKey = null;
+ Arrays.fill(keys, 0);
+ int[] rangesForMaskedByte =
+ getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+ try {
+ keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ maskedKey = getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys));
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Method will return the start key based on KeyGenerator for the respective
+ * filter resolved instance.
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return long[] start key
+ */
+ public static long[] getStartKey(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties, long[] startKey) {
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
+ dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ List<DimColumnFilterInfo> values = entry.getValue();
+ if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : values) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ getStartKeyBasedOnFilterResoverInfo(dimensionFilter, startKey);
+ }
+ return startKey;
+ }
+
+ /**
+ * Algorithm for getting the start key for a filter
+ * step 1: Iterate through each dimension and verify whether its not an exclude filter.
+ * step 2: Intialize start key with the first filter member value present in each filter model
+ * for the respective dimensions.
+ * step 3: since its a no dictionary start key there will only actual value so compare
+ * the first filter model value with respect to the dimension data type.
+ * step 4: The least value will be considered as the start key of dimension by comparing all
+ * its filter model.
+ * step 5: create a byte array of start key which comprises of least filter member value of
+ * all dimension and the indexes which will help to read the respective filter value.
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @param setOfStartKeyByteArray
+ * @return
+ */
+ public static void getStartKeyForNoDictionaryDimension(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
+ dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
+ // step 1
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
+ if (null == listOfDimColFilterInfo) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ // step 2
+ byte[] noDictionaryStartKey =
+ listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0);
+ if (setOfStartKeyByteArray.isEmpty()) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+
+ } else if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()),
+ noDictionaryStartKey) > 0) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ }
+ }
+ }
+ }
+
+ /**
+ * Algorithm for getting the end key for a filter
+ * step 1: Iterate through each dimension and verify whether its not an exclude filter.
+ * step 2: Initialize end key with the last filter member value present in each filter model
+ * for the respective dimensions.(Already filter models are sorted)
+ * step 3: since its a no dictionary end key there will only actual value so compare
+ * the last filter model value with respect to the dimension data type.
+ * step 4: The highest value will be considered as the end key of dimension by comparing all
+ * its filter model.
+ * step 5: create a byte array of end key which comprises of highest filter member value of
+ * all dimension and the indexes which will help to read the respective filter value.
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @param setOfEndKeyByteArray
+ * @return end key array
+ */
+ public static void getEndKeyForNoDictionaryDimension(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray) {
+
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
+ dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
+ // step 1
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
+ if (null == listOfDimColFilterInfo) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ // step 2
+ byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList()
+ .get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1);
+ if (setOfEndKeyByteArray.isEmpty()) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+
+ } else if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey)
+ < 0) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Method will pack all the byte[] to a single byte[] value by appending the
+ * indexes of the byte[] value which needs to be read. this method will be mailny used
+ * in case of no dictionary dimension processing for filters.
+ *
+ * @param noDictionaryValKeyList
+ * @return packed key with its indexes added in starting and its actual values.
+ */
+ private static byte[] getKeyWithIndexesAndValues(List<byte[]> noDictionaryValKeyList) {
+ ByteBuffer[] buffArr = new ByteBuffer[noDictionaryValKeyList.size()];
+ int index = 0;
+ for (byte[] singleColVal : noDictionaryValKeyList) {
+ buffArr[index] = ByteBuffer.allocate(singleColVal.length);
+ buffArr[index].put(singleColVal);
+ buffArr[index++].rewind();
+ }
+ // byteBufer.
+ return CarbonUtil.packByteBufferIntoSingleByteArray(buffArr);
+
+ }
+
+ /**
+ * This method will fill the start key array with the surrogate key present
+ * in filterinfo instance.
+ *
+ * @param dimensionFilter
+ * @param startKey
+ */
+ private static void getStartKeyBasedOnFilterResoverInfo(
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] startKey) {
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ List<DimColumnFilterInfo> values = entry.getValue();
+ if (null == values) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : values) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ for (DimColumnFilterInfo info : values) {
+ if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) {
+ startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0);
+ }
+ }
+ }
+ }
+
+ public static void getEndKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+ AbsoluteTableIdentifier tableIdentifier, long[] endKey, SegmentProperties segmentProperties)
+ throws QueryExecutionException {
+
+ List<CarbonDimension> updatedDimListBasedOnKeyGenerator =
+ getCarbonDimsMappedToKeyGenerator(segmentProperties.getDimensions());
+ for (int i = 0; i < endKey.length; i++) {
+ endKey[i] = getMaxValue(tableIdentifier, updatedDimListBasedOnKeyGenerator.get(i),
+ segmentProperties.getDimColumnsCardinality());
+ }
+ getEndKeyWithFilter(dimensionFilter, endKey);
+
+ }
+
+ private static List<CarbonDimension> getCarbonDimsMappedToKeyGenerator(
+ List<CarbonDimension> carbonDimensions) {
+ List<CarbonDimension> listOfCarbonDimPartOfKeyGen =
+ new ArrayList<CarbonDimension>(carbonDimensions.size());
+ for (CarbonDimension carbonDim : carbonDimensions) {
+ if (CarbonUtil.hasEncoding(carbonDim.getEncoder(), Encoding.DICTIONARY) || CarbonUtil
+ .hasEncoding(carbonDim.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+ listOfCarbonDimPartOfKeyGen.add(carbonDim);
+ }
+
+ }
+ return listOfCarbonDimPartOfKeyGen;
+ }
+
+ private static void getEndKeyWithFilter(
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] endKey) {
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ List<DimColumnFilterInfo> values = entry.getValue();
+ if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ continue;
+ }
+ boolean isExcludeFilterPresent = false;
+ for (DimColumnFilterInfo info : values) {
+ if (!info.isIncludeFilter()) {
+ isExcludeFilterPresent = true;
+ }
+ }
+ if (isExcludeFilterPresent) {
+ continue;
+ }
+
+ for (DimColumnFilterInfo info : values) {
+ if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList()
+ .get(info.getFilterList().size() - 1)) {
+ endKey[entry.getKey().getKeyOrdinal()] =
+ info.getFilterList().get(info.getFilterList().size() - 1);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * This API will get the max value of surrogate key which will be used for
+ * determining the end key of particular btree.
+ *
+ * @param dimCarinality
+ * @throws QueryExecutionException
+ */
+ private static long getMaxValue(AbsoluteTableIdentifier tableIdentifier,
+ CarbonDimension carbonDimension, int[] dimCarinality) throws QueryExecutionException {
+ // if (DataType.TIMESTAMP == carbonDimension.getDataType()) {
+ // return Integer.MAX_VALUE;
+ // }
+ // Get data from all the available slices of the table
+ if (null != dimCarinality) {
+ return dimCarinality[carbonDimension.getKeyOrdinal()];
+ }
+ return -1;
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param carbonDimension
+ * @return
+ * @throws QueryExecutionException
+ */
+ public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier,
+ CarbonDimension carbonDimension) throws QueryExecutionException {
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
+ new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(),
+ carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ Cache forwardDictionaryCache =
+ cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
+ // get the forward dictionary object
+ Dictionary forwardDictionary = null;
+ try {
+ forwardDictionary = (Dictionary) forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
+ } catch (CarbonUtilException e) {
+ throw new QueryExecutionException(e);
+ }
+ return forwardDictionary;
+ }
+
+ public static IndexKey createIndexKeyFromResolvedFilterVal(long[] startOrEndKey,
+ KeyGenerator keyGenerator, byte[] startOrEndKeyForNoDictDimension) {
+ IndexKey indexKey = null;
+ try {
+ indexKey =
+ new IndexKey(keyGenerator.generateKey(startOrEndKey), startOrEndKeyForNoDictDimension);
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ return indexKey;
+ }
+
+ /**
+ * API will create an filter executer tree based on the filter resolver
+ *
+ * @param filterExpressionResolverTree
+ * @param segmentProperties
+ * @return
+ */
+ public static FilterExecuter getFilterExecuterTree(
+ FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
+ Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ return createFilterExecuterTree(filterExpressionResolverTree, segmentProperties,
+ complexDimensionInfoMap);
+ }
+
+ /**
+ * API will prepare the Keys from the surrogates of particular filter resolver
+ *
+ * @param filterValues
+ * @param blockKeyGenerator
+ * @param dimension
+ * @param dimColumnExecuterInfo
+ */
+ public static void prepareKeysFromSurrogates(DimColumnFilterInfo filterValues,
+ KeyGenerator blockKeyGenerator, CarbonDimension dimension,
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
+ byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, blockKeyGenerator);
+ dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
+
+ }
+
+ /**
+ * method will create a default end key in case of no end key is been derived using existing
+ * filter or in case of non filter queries.
+ *
+ * @param segmentProperties
+ * @return
+ * @throws KeyGenException
+ */
+ public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
+ throws KeyGenException {
+ long[] dictionarySurrogateKey =
+ new long[segmentProperties.getDimensions().size() - segmentProperties
+ .getNumberOfNoDictionaryDimension()];
+ Arrays.fill(dictionarySurrogateKey, Long.MAX_VALUE);
+ IndexKey endIndexKey;
+ byte[] dictionaryendMdkey =
+ segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
+ endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
+ return endIndexKey;
+ }
+
+ public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+ // in case of non filter query when no dictionary columns are present we
+ // need to set the default end key, as for non filter query
+ // we need to get the last
+ // block of the btree so we are setting the max byte value in the end key
+ ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
+ (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
+ .getNumberOfNoDictionaryDimension());
+ // end key structure will be
+ //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
+ //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
+ // <second no dictionary column value> <N no dictionary column value>
+ //example if we have 2 no dictionary column
+ //<[0,4,0,5,127,127]>
+ short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryEndKeyBuffer.putShort((startPoint));
+ startPoint++;
+ }
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryEndKeyBuffer.put((byte) 127);
+ }
+ return noDictionaryEndKeyBuffer.array();
+ }
+
+ /**
+ * method will create a default end key in case of no end key is been
+ * derived using existing filter or in case of non filter queries.
+ *
+ * @param segmentProperties
+ * @return
+ * @throws KeyGenException
+ */
+ public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
+ throws KeyGenException {
+ IndexKey startIndexKey;
+ long[] dictionarySurrogateKey =
+ new long[segmentProperties.getDimensions().size() - segmentProperties
+ .getNumberOfNoDictionaryDimension()];
+ byte[] dictionaryStartMdkey =
+ segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
+
+ startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
+ return startIndexKey;
+ }
+
+ public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+ // in case of non filter query when no dictionary columns are present we
+ // need to set the default start key, as for non filter query we need to get the first
+ // block of the btree so we are setting the least byte value in the start key
+ ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
+ (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
+ .getNumberOfNoDictionaryDimension());
+ // end key structure will be
+ //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
+ //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
+ // <second no dictionary column value> <N no dictionary column value>
+ //example if we have 2 no dictionary column
+ //<[0,4,0,5,0,0]>
+ short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryStartKeyBuffer.putShort((startPoint));
+ startPoint++;
+ }
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryStartKeyBuffer.put((byte) 0);
+ }
+ return noDictionaryStartKeyBuffer.array();
+ }
+
+ public static int compareFilterKeyBasedOnDataType(String dictionaryVal, String memberVal,
+ DataType dataType) {
+ try {
+ switch (dataType) {
+ case SHORT:
+ return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
+ case INT:
+ return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
+ case DOUBLE:
+ return Double
+ .compare((Double.parseDouble(dictionaryVal)), (Double.parseDouble(memberVal)));
+ case LONG:
+ return Long.compare((Long.parseLong(dictionaryVal)), (Long.parseLong(memberVal)));
+ case BOOLEAN:
+ return Boolean
+ .compare((Boolean.parseBoolean(dictionaryVal)), (Boolean.parseBoolean(memberVal)));
+ case TIMESTAMP:
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ Date dateToStr;
+ Date dictionaryDate;
+ dateToStr = parser.parse(memberVal);
+ dictionaryDate = parser.parse(dictionaryVal);
+ return dictionaryDate.compareTo(dateToStr);
+
+ case DECIMAL:
+ java.math.BigDecimal javaDecValForDictVal = new java.math.BigDecimal(dictionaryVal);
+ java.math.BigDecimal javaDecValForMemberVal = new java.math.BigDecimal(memberVal);
+ return javaDecValForDictVal.compareTo(javaDecValForMemberVal);
+ default:
+ return -1;
+ }
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ /**
+ * method will set the start and end key for as per the filter resolver tree
+ * utilized visitor pattern inorder to populate the start and end key population.
+ *
+ * @param segmentProperties
+ * @param tableIdentifier
+ * @param filterResolver
+ * @param listOfStartEndKeys
+ * @throws QueryExecutionException
+ */
+ public static void traverseResolverTreeAndGetStartAndEndKey(SegmentProperties segmentProperties,
+ AbsoluteTableIdentifier tableIdentifier, FilterResolverIntf filterResolver,
+ List<IndexKey> listOfStartEndKeys) throws QueryExecutionException {
+ IndexKey searchStartKey = null;
+ IndexKey searchEndKey = null;
+ long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ List<byte[]> listOfStartKeyByteArray =
+ new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
+ List<byte[]> listOfEndKeyByteArray =
+ new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> defaultStartValues = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> defaultEndValues = new TreeMap<Integer, byte[]>();
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, tableIdentifier,
+ segmentProperties, startKey, setOfStartKeyByteArray, endKey, setOfEndKeyByteArray);
+ fillDefaultStartValue(defaultStartValues, segmentProperties);
+ fillDefaultEndValue(defaultEndValues, segmentProperties);
+ fillNullValuesStartIndexWithDefaultKeys(setOfStartKeyByteArray, segmentProperties);
+ fillNullValuesEndIndexWithDefaultKeys(setOfEndKeyByteArray, segmentProperties);
+ pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
+ pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
+
+ searchStartKey = FilterUtil
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
+
+ searchEndKey = FilterUtil
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
+ listOfStartEndKeys.add(searchStartKey);
+ listOfStartEndKeys.add(searchEndKey);
+
+ }
+
+ private static int compareFilterMembersBasedOnActualDataType(String filterMember1,
+ String filterMember2, DataType dataType) {
+ try {
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ case DOUBLE:
+
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ Double d1 = Double.parseDouble(filterMember1);
+ Double d2 = Double.parseDouble(filterMember2);
+ return d1.compareTo(d2);
+ case DECIMAL:
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ java.math.BigDecimal val1 = new BigDecimal(filterMember1);
+ java.math.BigDecimal val2 = new BigDecimal(filterMember2);
+ return val1.compareTo(val2);
+ case TIMESTAMP:
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ Date date1 = null;
+ Date date2 = null;
+ date1 = parser.parse(filterMember1);
+ date2 = parser.parse(filterMember2);
+ return date1.compareTo(date2);
+ case STRING:
+ default:
+ return filterMember1.compareTo(filterMember2);
+ }
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ private static void fillNullValuesStartIndexWithDefaultKeys(
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
+ }
+
+ }
+ }
+
+ private static void fillNullValuesEndIndexWithDefaultKeys(
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
+ }
+
+ }
+ }
+
+ private static void pruneStartAndEndKeys(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ List<byte[]> listOfStartKeyByteArray) {
+ for (Map.Entry<Integer, byte[]> entry : setOfStartKeyByteArray.entrySet()) {
+ listOfStartKeyByteArray.add(entry.getValue());
+ }
+ }
+
+ private static void fillDefaultStartValue(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
+ }
+
+ }
+
+ private static void fillDefaultEndValue(SortedMap<Integer, byte[]> setOfEndKeyByteArray,
+ SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ setOfEndKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
+ }
+ }
+
+ private static void traverseResolverTreeAndPopulateStartAndEndKeys(
+ FilterResolverIntf filterResolverTree, AbsoluteTableIdentifier tableIdentifier,
+ SegmentProperties segmentProperties, long[] startKeys,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray) throws QueryExecutionException {
+ if (null == filterResolverTree) {
+ return;
+ }
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(), tableIdentifier,
+ segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray);
+
+ filterResolverTree.getStartKey(segmentProperties, startKeys, setOfStartKeyByteArray);
+ filterResolverTree.getEndKey(segmentProperties, tableIdentifier, endKeys, setOfEndKeyByteArray);
+
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(), tableIdentifier,
+ segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray);
+ }
+
+ /**
+ * Method will find whether the expression needs to be resolved, this can happen
+ * if the expression is exclude and data type is null(mainly in IS NOT NULL filter scenario)
+ *
+ * @param rightExp
+ * @param isIncludeFilter
+ * @return
+ */
+ public static boolean isExpressionNeedsToResolved(Expression rightExp, boolean isIncludeFilter) {
+ if (!isIncludeFilter && rightExp instanceof LiteralExpression && (
+ DataType.NULL == ((LiteralExpression) rightExp)
+ .getLiteralExpDataType())) {
+ return true;
+ }
+ for (Expression child : rightExp.getChildren()) {
+ if (isExpressionNeedsToResolved(child, isIncludeFilter)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This method will print the error log.
+ *
+ * @param e
+ */
+ public static void logError(Throwable e, boolean invalidRowsPresent) {
+ if (!invalidRowsPresent) {
+ invalidRowsPresent = true;
+ LOGGER.error(e, CarbonCommonConstants.FILTER_INVALID_MEMBER + e.getMessage());
+ }
+ }
++
++ /**
++ * This method will return list of all the unknown expressions
++ *
++ * @param expression
++ */
++ public static List<UnknownExpression> getUnknownExpressionsList(Expression expression) {
++ List<UnknownExpression> listOfExp =
++ new ArrayList<UnknownExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
++ getUnknownExpressionsList(expression, listOfExp);
++ return listOfExp;
++ }
++
++ /**
++ * This method will prepare the list with all unknown expressions
++ *
++ * @param expression
++ * @param lst
++ */
++ private static void getUnknownExpressionsList(Expression expression,
++ List<UnknownExpression> lst) {
++ if (expression instanceof UnknownExpression) {
++ UnknownExpression colExp = (UnknownExpression) expression;
++ lst.add(colExp);
++ return;
++ }
++ for (Expression child : expression.getChildren()) {
++ getUnknownExpressionsList(child, lst);
++ }
++ }
++
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
index cde5c43,0000000..4f7d814
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
@@@ -1,52 -1,0 +1,52 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.filter.resolver;
+
+import java.util.SortedMap;
+
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.filter.intf.ExpressionType;
+
+public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
+
+ /**
+ *i
+ */
+ private static final long serialVersionUID = -761688076874662001L;
+
+ public AndFilterResolverImpl(FilterResolverIntf leftEvalutor, FilterResolverIntf rightEvalutor,
- ExpressionType filterExpressionType) {
- super(leftEvalutor, rightEvalutor, filterExpressionType);
++ ExpressionType filterExpressionType,BinaryExpression expression) {
++ super(leftEvalutor, rightEvalutor, expression);
+ }
+
+ @Override public void getStartKey(SegmentProperties segmentProperties, long[] startKeys,
+ SortedMap<Integer, byte[]> noDicStartKeys) {
+ leftEvalutor.getStartKey(segmentProperties, startKeys, noDicStartKeys);
+ rightEvalutor.getStartKey(segmentProperties, startKeys, noDicStartKeys);
+ }
+
+ @Override public void getEndKey(SegmentProperties segmentProperties,
+ AbsoluteTableIdentifier tableIdentifier, long[] endKeys,
+ SortedMap<Integer, byte[]> noDicEndKeys) throws QueryExecutionException {
+ leftEvalutor.getEndKey(segmentProperties, tableIdentifier, endKeys, noDicEndKeys);
+ rightEvalutor.getEndKey(segmentProperties, tableIdentifier, endKeys, noDicEndKeys);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
index 4fd9223,0000000..4aa448f
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
@@@ -1,111 -1,0 +1,113 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.scan.filter.resolver;
+
+import java.util.SortedMap;
+
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.Expression;
+import org.carbondata.scan.filter.intf.ExpressionType;
+import org.carbondata.scan.filter.intf.FilterExecuterType;
+import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class LogicalFilterResolverImpl implements FilterResolverIntf {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 5734382980564402914L;
+
+ protected FilterResolverIntf leftEvalutor;
+
+ protected FilterResolverIntf rightEvalutor;
+
+ protected ExpressionType filterExpressionType;
+
++ private BinaryExpression filterExpression;
++
+ public LogicalFilterResolverImpl(FilterResolverIntf leftEvalutor,
- FilterResolverIntf rightEvalutor, ExpressionType filterExpressionType) {
++ FilterResolverIntf rightEvalutor,BinaryExpression currentExpression) {
+ this.leftEvalutor = leftEvalutor;
+ this.rightEvalutor = rightEvalutor;
- this.filterExpressionType = filterExpressionType;
++ this.filterExpressionType = currentExpression.getFilterExpressionType();
++ this.filterExpression=currentExpression;
+ }
+
+ /**
+ * Logical filter resolver will return the left and right filter expresison
+ * node for filter evaluation, so in this instance no implementation is required.
+ *
+ * @param absoluteTableIdentifier
+ */
+ @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
+
+ }
+
+ /**
+ * Since its a binary condition expresion the getLeft method will get the left
+ * node of filter expression
+ *
+ * @return FilterResolverIntf.
+ */
+ public FilterResolverIntf getLeft() {
+ return leftEvalutor;
+ }
+
+ /**
+ * Since its a binary condition expresion the getRight method will get the left
+ * node of filter expression
+ *
+ * @return FilterResolverIntf.
+ */
+ public FilterResolverIntf getRight() {
+ return rightEvalutor;
+ }
+
+ @Override public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
+ return null;
+ }
+
+ @Override public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
+
+ }
+
+ @Override public void getEndKey(SegmentProperties segmentProperties,
+ AbsoluteTableIdentifier tableIdentifier, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray) throws QueryExecutionException {
+ }
+
+ @Override public FilterExecuterType getFilterExecuterType() {
+ switch (filterExpressionType) {
+ case OR:
+ return FilterExecuterType.OR;
+ case AND:
+ return FilterExecuterType.AND;
+
+ default:
+ return null;
+ }
+ }
+
+ @Override public Expression getFilterExpression() {
- // TODO Auto-generated method stub
- return null;
++ return filterExpression;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index a3e4ecf,0000000..5dd27ce
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@@ -1,102 -1,0 +1,110 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.carbondata.scan.expression.ColumnExpression;
+import org.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.carbondata.scan.filter.DimColumnFilterInfo;
+import org.carbondata.scan.filter.FilterUtil;
+import org.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorIntf {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CustomTypeDictionaryVisitor.class.getName());
+
+ /**
+ * This Visitor method is been used to resolve or populate the filter details
+ * by using custom type dictionary value, the filter membrers will be resolved using
+ * custom type function which will generate dictionary for the direct column type filter members
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ boolean isNotTimestampType = FilterUtil.checkIfDataTypeNotTimeStamp(metadata.getExpression());
+ resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
+ metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter(),
+ isNotTimestampType);
++ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
++ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
++ // Adding default surrogate key of null member inorder to not display the same while
++ // displaying the report as per hive compatibility.
++ resolvedFilterObject.getFilterList()
++ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
++ Collections.sort(resolvedFilterObject.getFilterList());
++ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+
+ private DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
+ AbsoluteTableIdentifier tableIdentifier, ColumnExpression columnExpression,
+ List<String> evaluateResultListFinal, boolean isIncludeFilter, boolean isNotTimestampType) {
+ List<Integer> surrogates = new ArrayList<Integer>(20);
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
+ // Reading the dictionary value direct
+ getSurrogateValuesForDictionary(evaluateResultListFinal, surrogates, isNotTimestampType,
+ directDictionaryGenerator);
+
+ Collections.sort(surrogates);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (surrogates.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterList(surrogates);
+ }
+ return columnFilterInfo;
+ }
+
+ private void getSurrogateValuesForDictionary(List<String> evaluateResultListFinal,
+ List<Integer> surrogates, boolean isNotTimestampType,
+ DirectDictionaryGenerator directDictionaryGenerator) {
+ String timeFormat = CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT;
+ if (isNotTimestampType) {
+ timeFormat = null;
+ }
+ for (String filterMember : evaluateResultListFinal) {
+ surrogates
+ .add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember, timeFormat));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index c133d3d,0000000..8bd45e3
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@@ -1,72 -1,0 +1,74 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
++import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.carbondata.scan.filter.DimColumnFilterInfo;
+import org.carbondata.scan.filter.FilterUtil;
+import org.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DictionaryColumnVisitor.class.getName());
+
+ /**
+ * This Visitor method is used to populate the visitableObj with direct dictionary filter details
+ * where the filters values will be resolve using dictionary cache.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ * @throws QueryExecutionException
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ try {
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList().add(1);
++ resolvedFilterObject.getFilterList()
++ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ } catch (QueryExecutionException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 4e938fd,885ad49..1d08fbd
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@@ -207,11 -201,12 +208,12 @@@ case class CarbonRelation
val dimensionsAttr = {
val sett = new LinkedHashSet(
- cubeMeta.carbonTable.getDimensionByTableName(cubeMeta.carbonTableIdentifier.getTableName)
+ tableMeta.carbonTable.getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName)
.asScala.asJava)
sett.asScala.toSeq.filter(!_.getColumnSchema.isInvisible).map(dim => {
- val output: DataType = metaData.carbonTable
- .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName).getDataType
+ val dimval = metaData.carbonTable
+ .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
+ val output: DataType = dimval.getDataType
.toString.toLowerCase match {
case "array" => CarbonMetastoreTypes
.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
[25/47] incubator-carbondata git commit: Problem: Null values getting
display in not equals filter for timestamp datatype column (#878)
Posted by ra...@apache.org.
Problem: Null values getting display in not equals filter for timestamp datatype column (#878)
Analysis: While setting surrogates for exclude filter in custom dictionary visitor, surrogate for null value is not getting added to surrogate list due to which the query displays null value in case of dictionary column filters.
Impact: Not equals, equals, not null and is null query on timestamp datatype column
Fix: Add surrogate value for null member in exclude filter list
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e367e12b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e367e12b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e367e12b
Branch: refs/heads/master
Commit: e367e12b1c2523f12887af03d2a87547d4e897ae
Parents: 82b7407
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Jul 27 20:30:49 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Wed Jul 27 20:30:49 2016 +0530
----------------------------------------------------------------------
.../visitor/CustomTypeDictionaryVisitor.java | 8 ++++++
.../visitor/DictionaryColumnVisitor.java | 4 ++-
.../spark/src/test/resources/datasample.csv | 1 +
.../dataload/TestLoadDataWithNoMeasure.scala | 2 +-
...estampDataTypeDirectDictionaryTestCase.scala | 29 ++++++++++++++++++--
...TypeDirectDictionaryWithNoDictTestCase.scala | 3 +-
6 files changed, 42 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index fa0f0e9..a2e5e21 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -64,6 +64,14 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter(),
isNotTimestampType);
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
visitableObj.setFilterValues(resolvedFilterObject);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 333a254..8488ad9 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
import org.carbondata.query.expression.exception.FilterIllegalMemberException;
import org.carbondata.query.expression.exception.FilterUnsupportedException;
@@ -61,7 +62,8 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList().add(1);
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
Collections.sort(resolvedFilterObject.getFilterList());
}
} catch (QueryExecutionException e) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/resources/datasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datasample.csv b/integration/spark/src/test/resources/datasample.csv
index fdf80e6..2f8ef35 100644
--- a/integration/spark/src/test/resources/datasample.csv
+++ b/integration/spark/src/test/resources/datasample.csv
@@ -1,3 +1,4 @@
empno,doj,salary
11,2016-04-14 15:00:09,5040.56
12,2016-03-14 15:00:09,1040.56
+13,,1040.56
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index f88e74c..9aa4189 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -48,7 +48,7 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("select empno from nomeasureTest"),
- Seq(Row("11"), Row("12"))
+ Seq(Row("11"), Row("12"), Row("13"))
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index baff810..f325254 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -70,15 +70,40 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
}
}
- test("select doj from directDictionaryCube") {
+ test("test direct dictionary for not null condition") {
checkAnswer(
- sql("select doj from directDictionaryCube"),
+ sql("select doj from directDictionaryCube where doj is not null"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
)
)
}
+ test("test direct dictionary for getting all the values") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+ Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+ Row(null)
+ )
+ )
+ }
+
+ test("test direct dictionary for not equals condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj != '2016-04-14 15:00:09.0'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+ )
+ )
+ }
+
+ test("test direct dictionary for null condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj is null"),
+ Seq(Row(null)
+ )
+ )
+ }
test("select doj from directDictionaryCube with equals filter") {
checkAnswer(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index a62c632..c654e95 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -74,7 +74,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with
checkAnswer(
sql("select doj from directDictionaryTable"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
- Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
+ Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+ Row(null)
)
)
}
[05/47] incubator-carbondata git commit: [Bug] Fix ColumnIdentifier
toString to let log information understandable. (#847)
Posted by ra...@apache.org.
[Bug] Fix ColumnIdentifier toString to let log information understandable. (#847)
[Bug] Fix ColumnIdentifier toString to let log information understandable. (#847)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/154a4d38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/154a4d38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/154a4d38
Branch: refs/heads/master
Commit: 154a4d388b96f5a338d17290f67a09345aebbc7a
Parents: 34741b4
Author: Zhangshunyu <zh...@huawei.com>
Authored: Fri Jul 22 20:09:10 2016 +0800
Committer: david <qi...@qq.com>
Committed: Fri Jul 22 20:09:10 2016 +0800
----------------------------------------------------------------------
.../main/java/org/carbondata/core/carbon/ColumnIdentifier.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/154a4d38/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java b/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
index f15aed3..74d8b4d 100644
--- a/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
+++ b/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
@@ -106,4 +106,8 @@ public class ColumnIdentifier implements Serializable {
return true;
}
+ @Override public String toString() {
+ return "ColumnIdentifier [columnId=" + columnId + "]";
+ }
+
}
[31/47] incubator-carbondata git commit: [CARBONDATA-111]Need to
catch exception in compaction and rethrow it to spark. (#858)
Posted by ra...@apache.org.
[CARBONDATA-111]Need to catch exception in compaction and rethrow it to spark. (#858)
Need to catch exception and rethrow it to spark. need to unlock the compaction lock in case of exception.
Need to set the property spark.job.interruptOnCancel should be set to true so that if the user kills the job in UI.
The task will also be killed. if any exception occurs in compaction then clean up the HDFS folders.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/973def99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/973def99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/973def99
Branch: refs/heads/master
Commit: 973def99f72c267e625497daf2ece18c18767d73
Parents: bdc1321
Author: ravikiran23 <ra...@gmail.com>
Authored: Thu Jul 28 21:03:20 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Thu Jul 28 21:03:20 2016 +0530
----------------------------------------------------------------------
.../spark/rdd/CarbonDataRDDFactory.scala | 88 ++++++------
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 136 ++++++++++---------
2 files changed, 122 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/973def99/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 15de7bf..54cf1bd 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -400,53 +400,44 @@ object CarbonDataRDDFactory extends Logging {
new Thread {
override def run(): Unit = {
- while (loadsToMerge.size() > 1) {
- // Deleting the any partially loaded data if present.
- // in some case the segment folder which is present in store will not have entry in
- // status.
- // so deleting those folders.
try {
- CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
- }
- catch {
- case e: Exception =>
- logger
- .error("Exception in compaction thread while clean up of stale segments " + e
- .getMessage
- )
- }
- val futureList: util.List[Future[Void]] = new util.ArrayList[Future[Void]](
- CarbonCommonConstants
- .DEFAULT_COLLECTION_SIZE
- )
+ while (loadsToMerge.size() > 1) {
+ deletePartialLoadsInCompaction(carbonLoadModel)
+ val futureList: util.List[Future[Void]] = new util.ArrayList[Future[Void]](
+ CarbonCommonConstants
+ .DEFAULT_COLLECTION_SIZE
+ )
- scanSegmentsAndSubmitJob(futureList)
+ scanSegmentsAndSubmitJob(futureList)
- futureList.asScala.foreach(future => {
- try {
+ futureList.asScala.foreach(future => {
future.get
}
- catch {
- case e: Exception =>
- logger.error("Exception in compaction thread " + e.getMessage)
- }
+ )
+ // scan again and deterrmine if anything is there to merge again.
+ readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
+ segList = carbonLoadModel.getLoadMetadataDetails
+
+ loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
+ hdfsStoreLocation,
+ carbonLoadModel,
+ partitioner.partitionCount,
+ compactionModel.compactionSize,
+ segList,
+ compactionModel.compactionType
+ )
}
- )
- // scan again and deterrmine if anything is there to merge again.
- readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
- segList = carbonLoadModel.getLoadMetadataDetails
-
- loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
- hdfsStoreLocation,
- carbonLoadModel,
- partitioner.partitionCount,
- compactionModel.compactionSize,
- segList,
- compactionModel.compactionType
- )
}
- executor.shutdown()
- compactionLock.unlock()
+ catch {
+ case e: Exception =>
+ logger.error("Exception in compaction thread " + e.getMessage)
+ throw e
+ }
+ finally {
+ executor.shutdownNow()
+ deletePartialLoadsInCompaction(carbonLoadModel)
+ compactionLock.unlock()
+ }
}
}.start
}
@@ -504,6 +495,23 @@ object CarbonDataRDDFactory extends Logging {
}
}
+ def deletePartialLoadsInCompaction(carbonLoadModel: CarbonLoadModel): Unit = {
+ // Deleting the any partially loaded data if present.
+ // in some case the segment folder which is present in store will not have entry in
+ // status.
+ // so deleting those folders.
+ try {
+ CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+ }
+ catch {
+ case e: Exception =>
+ logger
+ .error("Exception in compaction thread while clean up of stale segments " + e
+ .getMessage
+ )
+ }
+ }
+
def loadCarbonData(sc: SQLContext,
carbonLoadModel: CarbonLoadModel,
storeLocation: String,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/973def99/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
index 0129a2c..254d51b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -57,6 +57,7 @@ class CarbonMergerRDD[K, V](
val defaultParallelism = sc.defaultParallelism
sc.setLocalProperty("spark.scheduler.pool", "DDL")
+ sc.setLocalProperty("spark.job.interruptOnCancel", "true")
val storeLocation = carbonMergerMapping.storeLocation
val hdfsStoreLocation = carbonMergerMapping.hdfsStoreLocation
@@ -68,83 +69,94 @@ class CarbonMergerRDD[K, V](
override def compute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- var dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
- val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
- val tempLocationKey: String = carbonLoadModel.getDatabaseName + '_' + carbonLoadModel
- .getTableName + carbonLoadModel.getTaskNo
- CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
+ var mergeStatus = false
+ try {
+ var dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
+ val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
- // sorting the table block info List.
- var tableBlockInfoList = carbonSparkPartition.tableBlockInfos
+ val tempLocationKey: String = carbonLoadModel.getDatabaseName + '_' + carbonLoadModel
+ .getTableName + carbonLoadModel.getTaskNo
+ CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
- Collections.sort(tableBlockInfoList)
+ // sorting the table block info List.
+ var tableBlockInfoList = carbonSparkPartition.tableBlockInfos
- val segmentMapping: java.util.Map[String, TaskBlockInfo] =
- CarbonCompactionUtil.createMappingForSegments(tableBlockInfoList)
+ Collections.sort(tableBlockInfoList)
- val dataFileMetadataSegMapping: java.util.Map[String, List[DataFileFooter]] =
- CarbonCompactionUtil.createDataFileFooterMappingForSegments(tableBlockInfoList)
+ val segmentMapping: java.util.Map[String, TaskBlockInfo] =
+ CarbonCompactionUtil.createMappingForSegments(tableBlockInfoList)
- carbonLoadModel.setStorePath(hdfsStoreLocation)
+ val dataFileMetadataSegMapping: java.util.Map[String, List[DataFileFooter]] =
+ CarbonCompactionUtil.createDataFileFooterMappingForSegments(tableBlockInfoList)
- // taking the last table block info for getting the segment properties.
- val listMetadata = dataFileMetadataSegMapping.get(tableBlockInfoList.get
- (tableBlockInfoList.size()-1).getSegmentId())
+ carbonLoadModel.setStorePath(hdfsStoreLocation)
- val colCardinality: Array[Int] = listMetadata.get(listMetadata.size() - 1).getSegmentInfo
- .getColumnCardinality
+ // taking the last table block info for getting the segment properties.
+ val listMetadata = dataFileMetadataSegMapping.get(tableBlockInfoList.get
+ (tableBlockInfoList.size() - 1).getSegmentId()
+ )
- val segmentProperties = new SegmentProperties(
- listMetadata.get(listMetadata.size() - 1).getColumnInTable,
- colCardinality
- )
+ val colCardinality: Array[Int] = listMetadata.get(listMetadata.size() - 1).getSegmentInfo
+ .getColumnCardinality
- val exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties, schemaName,
- factTableName, hdfsStoreLocation, carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable,
- dataFileMetadataSegMapping
- )
+ val segmentProperties = new SegmentProperties(
+ listMetadata.get(listMetadata.size() - 1).getColumnInTable,
+ colCardinality
+ )
- // fire a query and get the results.
- var result2: util.List[RawResultIterator] = null
- try {
- result2 = exec.processTableBlocks()
- } catch {
- case e: Throwable =>
- exec.clearDictionaryFromQueryModel
- LOGGER.error(e)
- if (null != e.getMessage) {
- sys.error("Exception occurred in query execution :: " + e.getMessage)
- } else {
- sys.error("Exception occurred in query execution.Please check logs.")
- }
- }
+ val exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties, schemaName,
+ factTableName, hdfsStoreLocation, carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable,
+ dataFileMetadataSegMapping
+ )
- val mergeNumber = mergedLoadName
- .substring(mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
- CarbonCommonConstants.LOAD_FOLDER.length(), mergedLoadName.length()
+ // fire a query and get the results.
+ var result2: util.List[RawResultIterator] = null
+ try {
+ result2 = exec.processTableBlocks()
+ } catch {
+ case e: Throwable =>
+ exec.clearDictionaryFromQueryModel
+ LOGGER.error(e)
+ if (null != e.getMessage) {
+ sys.error("Exception occurred in query execution :: " + e.getMessage)
+ } else {
+ sys.error("Exception occurred in query execution.Please check logs.")
+ }
+ }
+
+ val mergeNumber = mergedLoadName
+ .substring(mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
+ CarbonCommonConstants.LOAD_FOLDER.length(), mergedLoadName.length()
+ )
+
+ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(schemaName,
+ factTableName,
+ carbonLoadModel.getTaskNo,
+ "0",
+ mergeNumber
)
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(schemaName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- mergeNumber
- )
+ carbonLoadModel.setSegmentId(mergeNumber)
+ carbonLoadModel.setPartitionId("0")
+ val merger =
+ new RowResultMerger(result2,
+ schemaName,
+ factTableName,
+ segmentProperties,
+ tempStoreLoc,
+ carbonLoadModel,
+ colCardinality
+ )
+ mergeStatus = merger.mergerSlice()
- carbonLoadModel.setSegmentId(mergeNumber)
- carbonLoadModel.setPartitionId("0")
- val merger =
- new RowResultMerger(result2,
- schemaName,
- factTableName,
- segmentProperties,
- tempStoreLoc,
- carbonLoadModel,
- colCardinality
- )
- val mergeStatus = merger.mergerSlice()
+ }
+ catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ throw e
+ }
var finished = false
[39/47] incubator-carbondata git commit: [CARBONDATA-121]Need to
identify the segments for merging each time after one compaction (#884)
Posted by ra...@apache.org.
[CARBONDATA-121]Need to identify the segments for merging each time after one compaction (#884)
as the user can delete some loads in between the compaction process.
* major should not run continuously and only one
* running the major compaction on all the segments which are present at the time of triggering the compaction.
then stopping the major compaction.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2d50d5c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2d50d5c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2d50d5c5
Branch: refs/heads/master
Commit: 2d50d5c5ac7a79d397e19baaaf1d1bfa9ea18006
Parents: f495b6b
Author: ravikiran23 <ra...@gmail.com>
Authored: Fri Jul 29 14:49:03 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jul 29 14:49:03 2016 +0530
----------------------------------------------------------------------
.../spark/merger/CarbonDataMergerUtil.java | 65 ++++++----
.../spark/rdd/CarbonDataRDDFactory.scala | 81 ++++++------
.../MajorCompactionStopsAfterCompaction.scala | 125 +++++++++++++++++++
3 files changed, 208 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2d50d5c5/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
index b290dd5..c2722a4 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -233,20 +233,8 @@ public final class CarbonDataMergerUtil {
List<LoadMetadataDetails> segments, CompactionType compactionType) {
List sortedSegments = new ArrayList(segments);
- // sort the segment details.
- Collections.sort(sortedSegments, new Comparator<LoadMetadataDetails>() {
- @Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
- double seg1Id = Double.parseDouble(seg1.getLoadName());
- double seg2Id = Double.parseDouble(seg2.getLoadName());
- if (seg1Id - seg2Id < 0) {
- return -1;
- }
- if (seg1Id - seg2Id > 0) {
- return 1;
- }
- return 0;
- }
- });
+
+ sortSegments(sortedSegments);
// check preserve property and preserve the configured number of latest loads.
@@ -273,6 +261,27 @@ public final class CarbonDataMergerUtil {
}
/**
+ * Sorting of the segments.
+ * @param segments
+ */
+ public static void sortSegments(List segments) {
+ // sort the segment details.
+ Collections.sort(segments, new Comparator<LoadMetadataDetails>() {
+ @Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
+ double seg1Id = Double.parseDouble(seg1.getLoadName());
+ double seg2Id = Double.parseDouble(seg2.getLoadName());
+ if (seg1Id - seg2Id < 0) {
+ return -1;
+ }
+ if (seg1Id - seg2Id > 0) {
+ return 1;
+ }
+ return 0;
+ }
+ });
+ }
+
+ /**
* This method will return the list of loads which are loaded at the same interval.
* This property is configurable.
*
@@ -691,16 +700,30 @@ public final class CarbonDataMergerUtil {
return combinedMap;
}
- public static List<LoadMetadataDetails> filterOutAlreadyMergedSegments(
- List<LoadMetadataDetails> segments, List<LoadMetadataDetails> loadsToMerge) {
+ /**
+ * Removing the already merged segments from list.
+ * @param segments
+ * @param loadsToMerge
+ * @return
+ */
+ public static List<LoadMetadataDetails> filterOutNewlyAddedSegments(
+ List<LoadMetadataDetails> segments, List<LoadMetadataDetails> loadsToMerge,
+ LoadMetadataDetails lastSeg) {
+
+ // take complete list of segments.
+ List<LoadMetadataDetails> list = new ArrayList<>(segments);
- ArrayList<LoadMetadataDetails> list = new ArrayList<>(segments);
+ List<LoadMetadataDetails> trimmedList =
+ new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- for (LoadMetadataDetails mergedSegs : loadsToMerge) {
- list.remove(mergedSegs);
- }
+ // sort list
+ CarbonDataMergerUtil.sortSegments(list);
- return list;
+ // first filter out newly added segments.
+ trimmedList = list.subList(0, list.indexOf(lastSeg) + 1);
+
+ return trimmedList;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2d50d5c5/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index ab77ea9..49e6702 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -397,6 +397,12 @@ object CarbonDataRDDFactory extends Logging {
if (loadsToMerge.size() > 1) {
+ val sortedSegments: util.List[LoadMetadataDetails] = new util.ArrayList[LoadMetadataDetails](
+ segList
+ )
+ CarbonDataMergerUtil.sortSegments(sortedSegments)
+ val lastSegment = sortedSegments.get(sortedSegments.size()-1)
+
new Thread {
override def run(): Unit = {
@@ -408,16 +414,24 @@ object CarbonDataRDDFactory extends Logging {
.DEFAULT_COLLECTION_SIZE
)
- scanSegmentsAndSubmitJob(futureList)
+ scanSegmentsAndSubmitJob(futureList, loadsToMerge)
futureList.asScala.foreach(future => {
future.get
}
)
- // scan again and deterrmine if anything is there to merge again.
+
+ // scan again and determine if anything is there to merge again.
readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
segList = carbonLoadModel.getLoadMetadataDetails
+ // in case of major compaction we will scan only once and come out as it will keep
+ // on doing major for the new loads also.
+ // excluding the newly added segments.
+ if (compactionModel.compactionType == CompactionType.MAJOR_COMPACTION) {
+ segList = CarbonDataMergerUtil
+ .filterOutNewlyAddedSegments(segList, loadsToMerge, lastSegment)
+ }
loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
hdfsStoreLocation,
carbonLoadModel,
@@ -446,52 +460,35 @@ object CarbonDataRDDFactory extends Logging {
}
/**
- * This will scan all the segments and submit the loads to be merged into the executor.
+ * This will submit the loads to be merged into the executor.
*
* @param futureList
*/
- def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]]): Unit = {
- breakable {
- while (true) {
+ def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]], loadsToMerge: util
+ .List[LoadMetadataDetails]): Unit = {
- val loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
- hdfsStoreLocation,
- carbonLoadModel,
- partitioner.partitionCount,
- compactionModel.compactionSize,
- segList,
- compactionModel.compactionType
- )
- if (loadsToMerge.size() > 1) {
- loadsToMerge.asScala.foreach(seg => {
- logger.info("load identified for merge is " + seg.getLoadName)
- }
- )
+ loadsToMerge.asScala.foreach(seg => {
+ logger.info("loads identified for merge is " + seg.getLoadName)
+ }
+ )
- val compactionCallableModel = CompactionCallableModel(hdfsStoreLocation,
- carbonLoadModel,
- partitioner,
- storeLocation,
- compactionModel.carbonTable,
- kettleHomePath,
- compactionModel.cubeCreationTime,
- loadsToMerge,
- sqlContext,
- compactionModel.compactionType)
+ val compactionCallableModel = CompactionCallableModel(hdfsStoreLocation,
+ carbonLoadModel,
+ partitioner,
+ storeLocation,
+ compactionModel.carbonTable,
+ kettleHomePath,
+ compactionModel.cubeCreationTime,
+ loadsToMerge,
+ sqlContext,
+ compactionModel.compactionType
+ )
- val future: Future[Void] = executor
- .submit(new CompactionCallable(compactionCallableModel
- )
- )
- futureList.add(future)
- segList = CarbonDataMergerUtil
- .filterOutAlreadyMergedSegments(segList, loadsToMerge)
- }
- else {
- break
- }
- }
- }
+ val future: Future[Void] = executor
+ .submit(new CompactionCallable(compactionCallableModel
+ )
+ )
+ futureList.add(future)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2d50d5c5/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
new file mode 100644
index 0000000..3cbf5dd
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
@@ -0,0 +1,125 @@
+package org.carbondata.spark.testsuite.datacompaction
+
+import java.io.File
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.util.CarbonProperties
+import org.carbondata.lcm.status.SegmentStatusManager
+import org.scalatest.BeforeAndAfterAll
+
+import scala.collection.JavaConverters._
+
+/**
+ * FT for compaction scenario where major compaction will only compact the segments which are
+ * present at the time of triggering the compaction.
+ */
+class MajorCompactionStopsAfterCompaction extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ sql("drop table if exists stopmajor")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+ sql(
+ "CREATE TABLE IF NOT EXISTS stopmajor (country String, ID Int, date Timestamp, name " +
+ "String, " +
+ "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
+ ".format'"
+ )
+
+
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+
+ val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv"
+ val csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
+
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE stopmajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE stopmajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ // compaction will happen here.
+ sql("alter table stopmajor compact 'major'"
+ )
+ Thread.sleep(2000)
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE stopmajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE stopmajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ if (checkCompactionCompletedOrNot("0.1")) {
+ }
+
+ }
+
+ /**
+ * Check if the compaction is completed or not.
+ *
+ * @param requiredSeg
+ * @return
+ */
+ def checkCompactionCompletedOrNot(requiredSeg: String): Boolean = {
+ var status = false
+ var noOfRetries = 0
+ while (!status && noOfRetries < 10) {
+
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "stopmajor", noOfRetries + "")
+ )
+ )
+ val segments = segmentStatusManager.getValidSegments().listOfValidSegments.asScala.toList
+ segments.foreach(seg =>
+ System.out.println( "valid segment is =" + seg)
+ )
+
+ if (!segments.contains(requiredSeg)) {
+ // wait for 2 seconds for compaction to complete.
+ System.out.println("sleping for 2 seconds.")
+ Thread.sleep(2000)
+ noOfRetries += 1
+ }
+ else {
+ status = true
+ }
+ }
+ return status
+ }
+
+ /**
+ * Test whether major compaction is not included in minor compaction.
+ */
+ test("delete merged folder and check segments") {
+ // delete merged segments
+ sql("clean files for table stopmajor")
+
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "stopmajor", "rrr")
+ )
+ )
+ // merged segment should not be there
+ val segments = segmentStatusManager.getValidSegments.listOfValidSegments.asScala.toList
+ assert(segments.contains("0.1"))
+ assert(!segments.contains("0.2"))
+ assert(!segments.contains("0"))
+ assert(!segments.contains("1"))
+ assert(segments.contains("2"))
+ assert(segments.contains("3"))
+
+ }
+
+ override def afterAll {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+
+}
[28/47] incubator-carbondata git commit: [CARBONDATA-95] Problem
Columns values with numeric data types are not getting parsed when included
in dictionary_include (#869)
Posted by ra...@apache.org.
[CARBONDATA-95] Problem Columns values with numeric data types are not getting parsed when included in dictionary_include (#869)
Analysis: When a numeric datatype lets say Decimal is defined for a column and the column is included as dictionary_include, then the whatever precision and scale is defined by the user is not taken into consideration and each value is accepted and dictionary is generated for that value.
Solution: Parse big decimal while generating global dictionary and dictionary look up and set the precision and scale specified by the user
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/adba5973
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/adba5973
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/adba5973
Branch: refs/heads/master
Commit: adba597364b418156c60c317fa99674dd7ad71d0
Parents: 6dfaefe
Author: manishgupta88 <to...@gmail.com>
Authored: Thu Jul 28 16:09:24 2016 +0530
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Thu Jul 28 16:09:24 2016 +0530
----------------------------------------------------------------------
.../processing/datatypes/PrimitiveDataType.java | 3 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 16 +++----
.../csvbased/CarbonCSVBasedSeqGenStep.java | 46 +++++++++++---------
3 files changed, 35 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/adba5973/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
index 3d272d5..d4b9b58 100644
--- a/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -163,8 +163,7 @@ public class PrimitiveDataType implements GenericDataType {
String[] delimiter, int delimiterIndex, DataOutputStream dataOutputStream,
CarbonCSVBasedDimSurrogateKeyGen surrogateKeyGen) throws KettleException, IOException {
String parsedValue = DataTypeUtil.parseValue(inputString,
- surrogateKeyGen.getDimensionNameToDimensionMapping()
- .get(tableName + CarbonCommonConstants.UNDERSCORE + name));
+ surrogateKeyGen.getDimensionOrdinalToDimensionMapping()[dimensionOrdinal]);
Integer surrogateKey = null;
if (null == parsedValue) {
surrogateKey = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/adba5973/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index 8fc1196..b72de8e 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -117,9 +117,9 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
new HashMap<String, Map<ArrayWrapper, Integer>>(
CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
/**
- * dimension name to dimension mapping
+ * dimension ordinal to dimension mapping
*/
- private Map<String, CarbonDimension> dimensionNameToDimensionMapping;
+ private CarbonDimension[] dimensionOrdinalToDimensionMapping;
/**
* rwLock2
*/
@@ -520,15 +520,15 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
/**
* @return
*/
- public Map<String, CarbonDimension> getDimensionNameToDimensionMapping() {
- return dimensionNameToDimensionMapping;
+ public CarbonDimension[] getDimensionOrdinalToDimensionMapping() {
+ return dimensionOrdinalToDimensionMapping;
}
/**
- * @param dimensionNameToDimensionMapping
+ * @param dimensionOrdinalToDimensionMapping
*/
- public void setDimensionNameToDimensionMapping(
- Map<String, CarbonDimension> dimensionNameToDimensionMapping) {
- this.dimensionNameToDimensionMapping = dimensionNameToDimensionMapping;
+ public void setDimensionOrdinalToDimensionMapping(
+ CarbonDimension[] dimensionOrdinalToDimensionMapping) {
+ this.dimensionOrdinalToDimensionMapping = dimensionOrdinalToDimensionMapping;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/adba5973/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 207e68d..d5ecb37 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -435,7 +435,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
if (null != getInputRowMeta()) {
generateNoDictionaryAndComplexIndexMapping();
data.getSurrogateKeyGen()
- .setDimensionNameToDimensionMapping(populateNameToCarbonDimensionMap());
+ .setDimensionOrdinalToDimensionMapping(populateNameToCarbonDimensionMap());
}
serializationNullFormat = meta.getTableOptionWrapper().get("serialization_null_format");
}
@@ -1147,10 +1147,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
surrogateKeyGen.max[m] = Integer.MAX_VALUE;
} else {
- String parsedValue = DataTypeUtil.parseValue(tuple,
- data.getSurrogateKeyGen().getDimensionNameToDimensionMapping().get(
- meta.getTableName() + CarbonCommonConstants.UNDERSCORE + columnName
- .toLowerCase()));
+ String parsedValue = DataTypeUtil.parseValue(tuple, data.getSurrogateKeyGen()
+ .getDimensionOrdinalToDimensionMapping()[memberMapping[i]]);
if(null == parsedValue) {
surrogateKeyForHrrchy[0] = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
} else {
@@ -1838,34 +1836,42 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
}
}
- private Map<String, CarbonDimension> populateNameToCarbonDimensionMap() {
+ private CarbonDimension[] populateNameToCarbonDimensionMap() {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
meta.getSchemaName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
List<CarbonDimension> dimensionsList = carbonTable.getDimensionByTableName(meta.getTableName());
- Map<String, CarbonDimension> dimensionNameToDimensionMapping =
- new HashMap<>(dimensionsList.size());
- for (CarbonDimension dimension : dimensionsList) {
+ CarbonDimension[] dimensionOrdinalToDimensionMapping =
+ new CarbonDimension[meta.getColumnSchemaDetailsWrapper().getColumnSchemaDetailsMap()
+ .size()];
+ List<CarbonDimension> dimListExcludingNoDictionaryColumn = dimensionsList;
+ if (null != meta.getNoDictionaryDims() && meta.getNoDictionaryDims().length() > 0) {
+ dimListExcludingNoDictionaryColumn =
+ new ArrayList<>(dimensionsList.size() - meta.noDictionaryCols.length);
+ for (CarbonDimension dimension : dimensionsList) {
+ if (!dimension.getEncoder().isEmpty()) {
+ dimListExcludingNoDictionaryColumn.add(dimension);
+ }
+ }
+ }
+ for (int i = 0; i < dimListExcludingNoDictionaryColumn.size(); i++) {
+ CarbonDimension dimension = dimListExcludingNoDictionaryColumn.get(meta.memberMapping[i]);
if (dimension.isComplex()) {
- populateComplexDimension(dimensionNameToDimensionMapping, dimension);
+ populateComplexDimension(dimensionOrdinalToDimensionMapping, dimension);
} else {
- dimensionNameToDimensionMapping.put(
- meta.getTableName() + CarbonCommonConstants.UNDERSCORE + dimension.getColName()
- .toLowerCase(), dimension);
+ dimensionOrdinalToDimensionMapping[meta.memberMapping[i]] = dimension;
}
}
- return dimensionNameToDimensionMapping;
+ return dimensionOrdinalToDimensionMapping;
}
- private void populateComplexDimension(
- Map<String, CarbonDimension> dimensionNameToDimensionMapping, CarbonDimension dimension) {
+ private void populateComplexDimension(CarbonDimension[] dimensionOrdinalToDimensionMapping,
+ CarbonDimension dimension) {
List<CarbonDimension> listOfChildDimensions = dimension.getListOfChildDimensions();
for (CarbonDimension childDimension : listOfChildDimensions) {
if (childDimension.isComplex()) {
- populateComplexDimension(dimensionNameToDimensionMapping, childDimension);
+ populateComplexDimension(dimensionOrdinalToDimensionMapping, childDimension);
} else {
- dimensionNameToDimensionMapping.put(
- meta.getTableName() + CarbonCommonConstants.UNDERSCORE + childDimension.getColName(),
- childDimension);
+ dimensionOrdinalToDimensionMapping[childDimension.getOrdinal()] = childDimension;
}
}
}
[33/47] incubator-carbondata git commit: [CARBONDATA-112]
Regexp_replace function is throwing NullPointerException (#867)
Posted by ra...@apache.org.
[CARBONDATA-112] Regexp_replace function is throwing NullPointerException (#867)
as the expression is evaluated on executor side in carbon, transient variable was not initialized.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/935e0d3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/935e0d3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/935e0d3a
Branch: refs/heads/master
Commit: 935e0d3a6dad7300adc980df6bfacc1272dff904
Parents: bf7f9f7
Author: nareshpr <pr...@gmail.com>
Authored: Fri Jul 29 00:21:39 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jul 29 00:21:39 2016 +0530
----------------------------------------------------------------------
.../filters/measurefilter/util/FilterUtil.java | 32 ++++++++++++++++++++
.../org/apache/spark/sql/CarbonOperators.scala | 8 ++++-
.../spark/sql/SparkUnknownExpression.scala | 20 +++++++++---
.../carbondata/spark/rdd/CarbonQueryRDD.scala | 17 +++++++++--
...estampDataTypeDirectDictionaryTestCase.scala | 17 +++++++++--
.../filterexpr/AllDataTypesTestCaseFilter.scala | 14 ++++++++-
6 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java b/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
index fb02b74..5ac89c6 100644
--- a/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
+++ b/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
@@ -71,6 +71,7 @@ import org.carbondata.query.expression.ColumnExpression;
import org.carbondata.query.expression.Expression;
import org.carbondata.query.expression.ExpressionResult;
import org.carbondata.query.expression.LiteralExpression;
+import org.carbondata.query.expression.UnknownExpression;
import org.carbondata.query.expression.conditional.ListExpression;
import org.carbondata.query.expression.exception.FilterIllegalMemberException;
import org.carbondata.query.expression.exception.FilterUnsupportedException;
@@ -1351,4 +1352,35 @@ public final class FilterUtil {
LOGGER.error(e, CarbonCommonConstants.FILTER_INVALID_MEMBER + e.getMessage());
}
}
+
+ /**
+ * This method will return list of all the unknown expressions
+ *
+ * @param expression
+ */
+ public static List<UnknownExpression> getUnknownExpressionsList(Expression expression) {
+ List<UnknownExpression> listOfExp =
+ new ArrayList<UnknownExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getUnknownExpressionsList(expression, listOfExp);
+ return listOfExp;
+ }
+
+ /**
+ * This method will prepare the list with all unknown expressions
+ *
+ * @param expression
+ * @param lst
+ */
+ private static void getUnknownExpressionsList(Expression expression,
+ List<UnknownExpression> lst) {
+ if (expression instanceof UnknownExpression) {
+ UnknownExpression colExp = (UnknownExpression) expression;
+ lst.add(colExp);
+ return;
+ }
+ for (Expression child : expression.getChildren()) {
+ getUnknownExpressionsList(child, lst);
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
index 76b7e8c..e4e7ea4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
@@ -570,10 +570,16 @@ case class CarbonTableScan(
conf,
cubeCreationTime,
schemaLastUpdatedTime,
- carbonCatalog.storePath)
+ carbonCatalog.storePath,
+ this)
big
}
+ def newProjection(expression: org.apache.spark.sql.catalyst.expressions.Expression):
+ InternalRow => Any = {
+ super.newProjection(Seq(expression), output)
+ }
+
def doExecute(): RDD[InternalRow] = {
def toType(obj: Any): Any = {
obj match {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index 058cea5..1162db4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -21,6 +21,7 @@ import java.util.{ArrayList, List}
import scala.collection.JavaConverters._
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
import org.carbondata.core.carbon.metadata.encoder.Encoding
@@ -30,9 +31,11 @@ import org.carbondata.query.expression.conditional.ConditionalExpression
import org.carbondata.query.expression.exception.FilterUnsupportedException
import org.carbondata.spark.util.CarbonScalaUtil
-class SparkUnknownExpression(sparkExp: SparkExpression)
+class SparkUnknownExpression(var sparkExp: SparkExpression)
extends UnknownExpression with ConditionalExpression {
+ private var evaluateExpression: (InternalRow) => Any = sparkExp.eval
+ private var isExecutor: Boolean = false
children.addAll(getColumnList())
override def evaluate(carbonRowInstance: RowIntf): ExpressionResult = {
@@ -49,10 +52,13 @@ class SparkUnknownExpression(sparkExp: SparkExpression)
}
}
try {
- val sparkRes = sparkExp.eval(
- new GenericMutableRow(values.map(a => a.asInstanceOf[Any]).toArray)
- )
-
+ val result = evaluateExpression(
+ new GenericMutableRow(values.map(a => a.asInstanceOf[Any]).toArray))
+ val sparkRes = if (isExecutor) {
+ result.asInstanceOf[InternalRow].get(0, sparkExp.dataType)
+ } else {
+ result
+ }
new ExpressionResult(CarbonScalaUtil.convertSparkToCarbonDataType(sparkExp.dataType),
sparkRes
)
@@ -70,6 +76,10 @@ class SparkUnknownExpression(sparkExp: SparkExpression)
sparkExp.toString()
}
+ def setEvaluateExpression(evaluateExpression: (InternalRow) => Any): Unit = {
+ this.evaluateExpression = evaluateExpression
+ isExecutor = true
+ }
def getColumnList(): java.util.List[ColumnExpression] = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
index 97f1993..50d3ca7 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
@@ -27,6 +27,8 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{Logging, Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.CarbonTableScan
+import org.apache.spark.sql.SparkUnknownExpression
import org.apache.spark.sql.hive.DistributionUtil
import org.carbondata.common.logging.LogServiceFactory
@@ -39,11 +41,11 @@ import org.carbondata.query.carbon.executor.QueryExecutorFactory
import org.carbondata.query.carbon.model.QueryModel
import org.carbondata.query.carbon.result.RowResult
import org.carbondata.query.expression.Expression
+import org.carbondata.query.filters.measurefilter.util.FilterUtil
import org.carbondata.spark.Value
import org.carbondata.spark.load.CarbonLoaderUtil
import org.carbondata.spark.util.QueryPlanUtil
-
class CarbonSparkPartition(rddId: Int, val idx: Int,
val locations: Array[String],
val tableBlockInfos: util.List[TableBlockInfo])
@@ -69,7 +71,8 @@ class CarbonQueryRDD[V: ClassTag](
@transient conf: Configuration,
cubeCreationTime: Long,
schemaLastUpdatedTime: Long,
- baseStoreLocation: String)
+ baseStoreLocation: String,
+ sparkPlan: CarbonTableScan = null)
extends RDD[V](sc, Nil) with Logging {
val defaultParallelism = sc.defaultParallelism
@@ -169,6 +172,16 @@ class CarbonQueryRDD[V: ClassTag](
override def compute(thepartition: Partition, context: TaskContext): Iterator[V] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val filterResolverTree = this.queryModel.getFilterExpressionResolverTree
+ if (null != filterResolverTree && null != sparkPlan) {
+ FilterUtil.getUnknownExpressionsList(filterResolverTree.getFilterExpression).
+ asScala.foreach(unknownExpression => {
+ val unKnownSparkExpression = unknownExpression.
+ asInstanceOf[org.apache.spark.sql.SparkUnknownExpression]
+ unKnownSparkExpression.setEvaluateExpression(
+ sparkPlan.newProjection(unKnownSparkExpression.sparkExp))
+ })
+ }
val iter = new Iterator[V] {
var rowIterator: CarbonIterator[Array[Object]] = _
var queryStartTime: Long = 0
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index f325254..db67c86 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -113,12 +113,25 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
}
- test("select doj from directDictionaryCube with greater than filter") {
+ test("select doj from directDictionaryCube with regexp_replace equals filter") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where regexp_replace(doj, '-', '/') = '2016/03/14 15:00:09'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09")))
+ )
+ }
+
+ test("select doj from directDictionaryCube with regexp_replace NOT IN filter") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')"),
+ Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")), Row(null))
+ )
+ }
+
+ test("select doj from directDictionaryCube with greater than filter") {
checkAnswer(
sql("select doj from directDictionaryCube where doj>'2016-03-14 15:00:09'"),
Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")))
)
-
}
test("select count(doj) from directDictionaryCube") {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/935e0d3a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
index d9abe75..7cb6dfd 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
@@ -46,7 +46,19 @@ class AllDataTypesTestCaseFilter extends QueryTest with BeforeAndAfterAll {
sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypescubeFilter where empname in ('arvind','ayushi') group by empno,empname,utilization"),
sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypescubeFilter_hive where empname in ('arvind','ayushi') group by empno,empname,utilization"))
}
-
+
+ test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')") {
+ checkAnswer(
+ sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"),
+ sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"))
+ }
+
+ test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'") {
+ checkAnswer(
+ sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"),
+ sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"))
+ }
+
override def afterAll {
sql("drop table alldatatypescubeFilter")
sql("drop table alldatatypescubeFilter_hive")
[43/47] incubator-carbondata git commit: Merge remote-tracking branch
'carbon_master/master' into apache/master
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 74239a3,5c25406..22b6021
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@@ -31,7 -32,8 +32,8 @@@ import org.apache.spark.sql.catalyst.{S
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+ import org.apache.spark.sql.execution.ExplainCommand
-import org.apache.spark.sql.execution.command.{DimensionRelation, _}
+import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.datasources.DescribeCommand
import org.apache.spark.sql.hive.HiveQlWrapper
@@@ -332,109 -340,122 +340,123 @@@ class CarbonSqlParser(
// if create table taken is found then only we will handle.
case Token("TOK_CREATETABLE", children) =>
- var fields: Seq[Field] = Seq[Field]()
- var tableComment: String = ""
- var tableProperties = Map[String, String]()
- var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
- var likeTableName: String = ""
- var storedBy: String = ""
- var ifNotExistPresent: Boolean = false
- var dbName: Option[String] = None
- var tableName: String = ""
- children.collect {
- // collecting all the field list
- case list@Token("TOK_TABCOLLIST", _) =>
- val cols = BaseSemanticAnalyzer.getColumns(list, true)
- if (cols != null) {
- val dupColsGrp = cols.asScala
- .groupBy(x => x.getName) filter { case (_, colList) => colList
- .size > 1
- }
- if (dupColsGrp.size > 0) {
- var columnName: String = ""
- dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
- columnName = columnName.substring(0, columnName.lastIndexOf(", "))
- val errorMessage = "Duplicate column name: " + columnName + " found in table " +
- ".Please check create table statement."
- throw new MalformedCarbonCommandException(errorMessage)
- }
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val name = Option(col.getName())
- // This is to parse complex data types
- val x = col.getName + ' ' + col.getType
- val f: Field = anyFieldDef(new lexical.Scanner(x))
- match {
- case Success(field, _) => field
- case failureOrError => new Field(columnName, dataType, name, None, null,
- Some("columnar"))
+ var fields: Seq[Field] = Seq[Field]()
+ var tableComment: String = ""
+ var tableProperties = Map[String, String]()
+ var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
+ var likeTableName: String = ""
+ var storedBy: String = ""
+ var ifNotExistPresent: Boolean = false
+ var dbName: Option[String] = None
+ var tableName: String = ""
+
+ try {
+
+ children.collect {
+ // collecting all the field list
+ case list@Token("TOK_TABCOLLIST", _) =>
+ val cols = BaseSemanticAnalyzer.getColumns(list, true)
+ if (cols != null) {
+ val dupColsGrp = cols.asScala.groupBy(x => x.getName) filter {
+ case (_, colList) => colList.size > 1
+ }
+ if (dupColsGrp.size > 0) {
+ var columnName: String = ""
+ dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
+ columnName = columnName.substring(0, columnName.lastIndexOf(", "))
+ val errorMessage = "Duplicate column name: " + columnName + " found in table " +
+ ".Please check create table statement."
+ throw new MalformedCarbonCommandException(errorMessage)
}
- // the data type of the decimal type will be like decimal(10,0)
- // so checking the start of the string and taking the precision and scale.
- // resetting the data type with decimal
- if (f.dataType.getOrElse("").startsWith("decimal")) {
- val (precision, scale) = getScaleAndPrecision(col.getType)
- f.precision = precision
- f.scale = scale
- f.dataType = Some("decimal")
+ cols.asScala.map { col =>
+ val columnName = col.getName()
+ val dataType = Option(col.getType)
+ val name = Option(col.getName())
+ // This is to parse complex data types
+ val x = col.getName + ' ' + col.getType
+ val f: Field = anyFieldDef(new lexical.Scanner(x))
+ match {
+ case Success(field, _) => field
+ case failureOrError => new Field(columnName, dataType, name, None, null,
+ Some("columnar"))
+ }
+ // the data type of the decimal type will be like decimal(10,0)
+ // so checking the start of the string and taking the precision and scale.
+ // resetting the data type with decimal
+ if (f.dataType.getOrElse("").startsWith("decimal")) {
+ val (precision, scale) = getScaleAndPrecision(col.getType)
+ f.precision = precision
+ f.scale = scale
+ f.dataType = Some("decimal")
+ }
+ fields ++= Seq(f)
}
- fields ++= Seq(f)
}
- }
- case Token("TOK_IFNOTEXISTS", _) =>
- ifNotExistPresent = true
-
- case t@Token("TOK_TABNAME", _) =>
- val (db, tblName) = extractDbNameTableName(t)
- dbName = db
- tableName = tblName.toLowerCase()
-
- case Token("TOK_TABLECOMMENT", child :: Nil) =>
- tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
- case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
- val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
- if (cols != null) {
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val comment = col.getComment
- val partitionCol = new PartitionerField(columnName, dataType, comment)
- partitionCols ++= Seq(partitionCol)
+ case Token("TOK_IFNOTEXISTS", _) =>
+ ifNotExistPresent = true
+
+ case t@Token("TOK_TABNAME", _) =>
+ val (db, tblName) = extractDbNameTableName(t)
+ dbName = db
+ tableName = tblName.toLowerCase()
+
+ case Token("TOK_TABLECOMMENT", child :: Nil) =>
+ tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
+
+ case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
+ val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
+ if (cols != null) {
+ cols.asScala.map { col =>
+ val columnName = col.getName()
+ val dataType = Option(col.getType)
+ val comment = col.getComment
+ val partitionCol = new PartitionerField(columnName, dataType, comment)
+ partitionCols ++= Seq(partitionCol)
+ }
}
- }
- case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
- tableProperties ++= getProperties(list)
+ case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
+ tableProperties ++= getProperties(list)
- case Token("TOK_LIKETABLE", child :: Nil) =>
- likeTableName = child.getChild(0).getText()
+ case Token("TOK_LIKETABLE", child :: Nil) =>
+ likeTableName = child.getChild(0).getText()
- case Token("TOK_STORAGEHANDLER", child :: Nil) =>
- storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
+ case Token("TOK_STORAGEHANDLER", child :: Nil) =>
+ storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
- case _ => // Unsupport features
- }
+ case _ => // Unsupport features
+ }
- if (!storedBy.equals(CarbonContext.datasourceName)) {
- // TODO: should execute by Hive instead of error
- sys.error("Not a carbon format request")
- }
+ if (!(storedBy.equals(CarbonContext.datasourceName) ||
+ storedBy.equals(CarbonContext.datasourceShortName))) {
+ // TODO: should execute by Hive instead of error
+ sys.error("Not a carbon format request")
+ }
- // validate tblProperties
- if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
- throw new MalformedCarbonCommandException("Invalid table properties")
- }
- // prepare table model of the collected tokens
- val tableModel: tableModel = prepareTableModel(ifNotExistPresent, dbName, tableName, fields,
- partitionCols,
- tableProperties)
-
- // get logical plan.
- CreateTable(tableModel)
+ // validate tblProperties
+ if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
+ throw new MalformedCarbonCommandException("Invalid table properties")
+ }
+ // prepare table model of the collected tokens
+ val tableModel: tableModel = prepareTableModel(ifNotExistPresent,
+ dbName,
+ tableName,
+ fields,
+ partitionCols,
+ tableProperties)
+
+ // get logical plan.
+ CreateTable(tableModel)
+ }
+ catch {
+ case ce: MalformedCarbonCommandException =>
+ val message = if (tableName.isEmpty) "Create table command failed. "
+ else if (!dbName.isDefined) s"Create table command failed for $tableName. "
+ else s"Create table command failed for ${dbName.get}.$tableName. "
+ LOGGER.audit(message + ce.getMessage)
+ throw ce
+ }
}
}
@@@ -1295,7 -1343,16 +1324,16 @@@
protected lazy val cleanFiles: Parser[LogicalPlan] =
CLEAN ~> FILES ~> FOR ~> TABLE ~> (ident <~ ".").? ~ ident <~ opt(";") ^^ {
- case schemaName ~ cubeName => CleanFiles(schemaName, cubeName.toLowerCase())
+ case databaseName ~ tableName => CleanFiles(databaseName, tableName.toLowerCase())
}
+ protected lazy val explainPlan: Parser[LogicalPlan] =
+ (EXPLAIN ~> opt(EXTENDED)) ~ startCommand ^^ {
+ case isExtended ~ logicalPlan =>
+ logicalPlan match {
+ case plan: CreateTable => ExplainCommand(logicalPlan, extended = isExtended.isDefined)
+ case _ => ExplainCommand(OneRowRelation)
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index e8f6e11,1162db4..9a16f77
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@@ -24,15 -25,17 +25,17 @@@ import org.apache.spark.sql.catalyst.In
import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
import org.carbondata.core.carbon.metadata.encoder.Encoding
-import org.carbondata.query.carbonfilterinterface.{ExpressionType, RowIntf}
-import org.carbondata.query.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
-import org.carbondata.query.expression.conditional.ConditionalExpression
-import org.carbondata.query.expression.exception.FilterUnsupportedException
+import org.carbondata.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
+import org.carbondata.scan.expression.conditional.ConditionalExpression
+import org.carbondata.scan.expression.exception.FilterUnsupportedException
+import org.carbondata.scan.filter.intf.{ExpressionType, RowIntf}
import org.carbondata.spark.util.CarbonScalaUtil
- class SparkUnknownExpression(sparkExp: SparkExpression)
+ class SparkUnknownExpression(var sparkExp: SparkExpression)
extends UnknownExpression with ConditionalExpression {
+ private var evaluateExpression: (InternalRow) => Any = sparkExp.eval
+ private var isExecutor: Boolean = false
children.addAll(getColumnList())
override def evaluate(carbonRowInstance: RowIntf): ExpressionResult = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index d6cbaf4,1dd066f..01bb218
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@@ -156,8 -157,13 +156,13 @@@ case class AlterTableModel(dbName: Opti
case class CompactionModel(compactionSize: Long,
compactionType: CompactionType,
carbonTable: CarbonTable,
- cubeCreationTime: Long)
+ tableCreationTime: Long)
+ case class CompactionCallableModel(hdfsStoreLocation: String, carbonLoadModel: CarbonLoadModel,
+ partitioner: Partitioner, storeLocation: String, carbonTable: CarbonTable, kettleHomePath: String,
+ cubeCreationTime: Long, loadsToMerge: util.List[LoadMetadataDetails], sqlContext: SQLContext,
+ compactionType: CompactionType)
+
object TableNewProcessor {
def apply(cm: tableModel, sqlContext: SQLContext): TableInfo = {
new TableNewProcessor(cm, sqlContext).process
@@@ -767,12 -1201,12 +772,12 @@@ private[sql] case class AlterTableCompa
def run(sqlContext: SQLContext): Seq[Row] = {
// TODO : Implement it.
- var tableName = alterTableModel.tableName
+ val tableName = alterTableModel.tableName
- val schemaName = getDB.getDatabaseName(alterTableModel.dbName, sqlContext)
+ val databaseName = getDB.getDatabaseName(alterTableModel.dbName, sqlContext)
if (null == org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
- .getCarbonTable(schemaName + "_" + tableName)) {
- logError("alter table failed. table not found: " + schemaName + "." + tableName)
- sys.error("alter table failed. table not found: " + schemaName + "." + tableName)
+ .getCarbonTable(databaseName + "_" + tableName)) {
+ logError("alter table failed. table not found: " + databaseName + "." + tableName)
+ sys.error("alter table failed. table not found: " + databaseName + "." + tableName)
}
val relation =
@@@ -975,20 -1417,23 +980,20 @@@ private[sql] case class DeleteLoadsByLo
throw new MalformedCarbonCommandException(errorMessage)
}
- var carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
+ val carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
- .getCarbonTable(schemaName + '_' + tableName)
+ .getCarbonTable(dbName + '_' + tableName)
- var segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
+ val segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
if (null == carbonTable) {
- var relation = CarbonEnv.getInstance(sqlContext).carbonCatalog.lookupRelation1(
- Option(schemaName),
- tableName,
- None
- )(sqlContext).asInstanceOf[CarbonRelation]
+ var relation = CarbonEnv.getInstance(sqlContext).carbonCatalog
+ .lookupRelation1(identifier, None)(sqlContext).asInstanceOf[CarbonRelation]
}
- var path = carbonTable.getMetaDataFilepath()
+ val path = carbonTable.getMetaDataFilepath()
- var invalidLoadTimestamps = segmentStatusManager
+ val invalidLoadTimestamps = segmentStatusManager
.updateDeletionStatus(loadDate, path, timeObj.asInstanceOf[java.lang.Long]).asScala
if(invalidLoadTimestamps.isEmpty) {
- LOGGER.audit(s"Delete load by load date is successfull for $schemaName.$tableName.")
+ LOGGER.audit(s"Delete load by load date is successfull for $dbName.$tableName.")
}
else {
sys.error("Delete load by load date is failed. No matching load found.")
@@@ -1491,17 -2116,16 +1503,17 @@@ private[sql] case class DeleteLoadByDat
def run(sqlContext: SQLContext): Seq[Row] = {
- val schemaName = getDB.getDatabaseName(schemaNameOp, sqlContext)
- LOGGER.audit(s"The delete load by date request has been received for $schemaName.$cubeName")
+ val dbName = getDB.getDatabaseName(databaseNameOp, sqlContext)
+ LOGGER.audit(s"The delete load by date request has been received for $dbName.$tableName")
+ val identifier = TableIdentifier(tableName, Option(dbName))
val relation = CarbonEnv.getInstance(sqlContext).carbonCatalog
- .lookupRelation1(Some(schemaName), cubeName, None)(sqlContext).asInstanceOf[CarbonRelation]
+ .lookupRelation1(identifier)(sqlContext).asInstanceOf[CarbonRelation]
var level: String = ""
- var carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata
+ val carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata
- .getInstance().getCarbonTable(schemaName + '_' + cubeName)
+ .getInstance().getCarbonTable(dbName + '_' + tableName)
if (relation == null) {
- LOGGER.audit(s"The delete load by date is failed. Table $schemaName.$cubeName does not exist")
- sys.error(s"Table $schemaName.$cubeName does not exist")
+ LOGGER.audit(s"The delete load by date is failed. Table $dbName.$tableName does not exist")
+ sys.error(s"Table $dbName.$tableName does not exist")
}
val matches: Seq[AttributeReference] = relation.dimensionsAttr.filter(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index 07b505c,e18e1d3..0775fea
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@@ -155,11 -207,19 +155,19 @@@ class CarbonMetastoreCatalog(hiveContex
// creating zookeeper instance once.
// if zookeeper is configured as carbon lock type.
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT)
- .equalsIgnoreCase(CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER)) {
- val zookeeperUrl = hiveContext.getConf("spark.deploy.zookeeper.url", "127.0.0.1:2181")
- val zookeeperUrl: String = hive.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
++ val zookeeperUrl: String = hiveContext.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
+ if (zookeeperUrl != null) {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperUrl)
ZookeeperInit.getInstance(zookeeperUrl)
+ LOGGER.info("Zookeeper url is configured. Taking the zookeeper as lock type.")
+ var configuredLockType = CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.LOCK_TYPE)
+ if (null == configuredLockType) {
+ configuredLockType = CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.LOCK_TYPE,
+ configuredLockType)
+ }
}
if (metadataPath == null) {
@@@ -429,25 -489,16 +437,16 @@@
}
}
- metadata.cubesMeta -= metadata.cubesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(schemaName) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(cubeName))(0)
+ metadata.tablesMeta -= metadata.tablesMeta.filter(
+ c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(dbName) &&
+ c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))(0)
org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
- .removeTable(schemaName + "_" + cubeName)
+ .removeTable(dbName + "_" + tableName)
- try {
- sqlContext.sql(s"DROP TABLE $dbName.$tableName").collect()
- } catch {
- case e: Exception =>
- LOGGER.audit(
- s"Error While deleting the table $dbName.$tableName during drop Table" + e.getMessage)
- }
- logInfo(s"Table $tableName of $dbName Database dropped syccessfully.")
- LOGGER.info("Table " + tableName + " of " + dbName + " Database dropped syccessfully.")
-
- sqlContext.asInstanceOf[HiveContext].runSqlHive(s"DROP TABLE IF EXISTS $schemaName.$cubeName")
++ sqlContext.asInstanceOf[HiveContext].runSqlHive(s"DROP TABLE IF EXISTS $dbName.$tableName")
}
- private def getTimestampFileAndType(schemaName: String, cubeName: String) = {
+ private def getTimestampFileAndType(databaseName: String, tableName: String) = {
val timestampFile = storePath + "/" + CarbonCommonConstants.SCHEMAS_MODIFIED_TIME_FILE
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 85c6a1d,49e6702..748a408
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@@ -405,44 -464,48 +410,48 @@@ object CarbonDataRDDFactory extends Log
*
* @param futureList
*/
- def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]]): Unit = {
- breakable {
- while (true) {
-
- val loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
- hdfsStoreLocation,
- carbonLoadModel,
- partitioner.partitionCount,
- compactionModel.compactionSize,
- segList,
- compactionModel.compactionType
- )
- if (loadsToMerge.size() > 1) {
- loadsToMerge.asScala.foreach(seg => {
- logger.info("load identified for merge is " + seg.getLoadName)
- }
- )
+ def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]], loadsToMerge: util
+ .List[LoadMetadataDetails]): Unit = {
- val future: Future[Void] = executor.submit(new CompactionCallable(hdfsStoreLocation,
- carbonLoadModel,
- partitioner,
- storeLocation,
- compactionModel.carbonTable,
- kettleHomePath,
- compactionModel.tableCreationTime,
- loadsToMerge,
- sqlContext
- )
- )
- futureList.add(future)
- segList = CarbonDataMergerUtil
- .filterOutAlreadyMergedSegments(segList, loadsToMerge)
- }
- else {
- break
- }
- }
+ loadsToMerge.asScala.foreach(seg => {
+ logger.info("loads identified for merge is " + seg.getLoadName)
}
+ )
+
+ val compactionCallableModel = CompactionCallableModel(hdfsStoreLocation,
+ carbonLoadModel,
+ partitioner,
+ storeLocation,
+ compactionModel.carbonTable,
+ kettleHomePath,
- compactionModel.cubeCreationTime,
++ compactionModel.tableCreationTime,
+ loadsToMerge,
+ sqlContext,
+ compactionModel.compactionType
+ )
+
+ val future: Future[Void] = executor
+ .submit(new CompactionCallable(compactionCallableModel
+ )
+ )
+ futureList.add(future)
+ }
+ }
+
+ def deletePartialLoadsInCompaction(carbonLoadModel: CarbonLoadModel): Unit = {
+ // Deleting the any partially loaded data if present.
+ // in some case the segment folder which is present in store will not have entry in
+ // status.
+ // so deleting those folders.
+ try {
+ CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+ }
+ catch {
+ case e: Exception =>
+ logger
+ .error("Exception in compaction thread while clean up of stale segments " + e
+ .getMessage
+ )
}
}
@@@ -713,50 -776,59 +722,59 @@@
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(hdfsStoreLocation,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName,
partitioner.partitionCount, currentLoadCount.toString)
- val status = new
- CarbonDataLoadRDD(sqlContext.sparkContext,
- new DataLoadResultImpl(),
- carbonLoadModel,
- storeLocation,
- hdfsStoreLocation,
- kettleHomePath,
- partitioner,
- columinar,
- currentRestructNumber,
- currentLoadCount,
- tableCreationTime,
- schemaLastUpdatedTime,
- blocksGroupBy,
- isTableSplitPartition
- ).collect()
- val newStatusMap = scala.collection.mutable.Map.empty[String, String]
- status.foreach { eachLoadStatus =>
- val state = newStatusMap.get(eachLoadStatus._1)
- state match {
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
- if eachLoadStatus._2.getLoadStatus == CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- case _ =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- }
- }
-
var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
- newStatusMap.foreach {
- case (key, value) =>
- if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
- !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
+ var status: Array[(String, LoadMetadataDetails)] = null
+ try {
+ status = new
- CarbonDataLoadRDD(sc.sparkContext,
++ CarbonDataLoadRDD(sqlContext.sparkContext,
+ new DataLoadResultImpl(),
+ carbonLoadModel,
+ storeLocation,
+ hdfsStoreLocation,
+ kettleHomePath,
+ partitioner,
+ columinar,
+ currentRestructNumber,
+ currentLoadCount,
- cubeCreationTime,
++ tableCreationTime,
+ schemaLastUpdatedTime,
+ blocksGroupBy,
+ isTableSplitPartition
+ ).collect()
+ val newStatusMap = scala.collection.mutable.Map.empty[String, String]
+ status.foreach { eachLoadStatus =>
+ val state = newStatusMap.get(eachLoadStatus._1)
+ state match {
+ case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
+ case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
+ if eachLoadStatus._2.getLoadStatus ==
+ CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
+ case _ =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
}
- }
+ }
- if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
- partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
- loadStatus = partitionStatus
+ newStatusMap.foreach {
+ case (key, value) =>
+ if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
+ !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
+ }
+ }
+
+ if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
+ partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
+ loadStatus = partitionStatus
+ }
+ } catch {
+ case ex: Throwable =>
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ logInfo("DataLoad failure")
+ logger.error(ex)
}
if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
@@@ -779,10 -851,12 +797,10 @@@
aggTables.asScala.foreach { aggTableName =>
CarbonLoaderUtil
.deleteSlice(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName, aggTableName, hdfsStoreLocation,
- currentRestructNumber, newSlice
- )
+ carbonLoadModel.getTableName, hdfsStoreLocation, currentRestructNumber, newSlice)
}
}
- message = "Dataload failure"
+ message = "DataLoad failure"
}
logInfo("********clean up done**********")
logger.audit(s"Data load is failed for " +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 78538f1,70d0cb1..e79937f
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@@ -35,8 -36,10 +36,11 @@@ import org.carbondata.core.carbon.{Carb
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
+ import org.carbondata.core.locks.CarbonLockFactory
+ import org.carbondata.core.locks.LockUsage
+ import org.carbondata.core.util.CarbonProperties
import org.carbondata.core.util.CarbonTimeStatisticsFactory
+import org.carbondata.lcm.locks.{CarbonLockFactory, LockUsage}
import org.carbondata.spark.load.{CarbonLoaderUtil, CarbonLoadModel}
import org.carbondata.spark.partition.reader.{CSVParser, CSVReader}
import org.carbondata.spark.tasks.DictionaryWriterTask
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
index e22869c,7b15cbf..f687006
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
@@@ -67,83 -70,119 +69,119 @@@ class CarbonMergerRDD[K, V]
override def compute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
- var dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
- val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
- val tempLocationKey: String = carbonLoadModel.getDatabaseName + '_' + carbonLoadModel
- .getTableName + carbonLoadModel.getTaskNo
+ carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
+ val tempLocationKey: String = CarbonCommonConstants
+ .COMPACTION_KEY_WORD + '_' + carbonLoadModel
+ .getDatabaseName + '_' + carbonLoadModel
+ .getTableName + '_' + carbonLoadModel.getTaskNo
+
+ val storeLocations = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (null != storeLocations && storeLocations.length > 0) {
+ storeLocation = storeLocations(Random.nextInt(storeLocations.length))
+ }
+ if (storeLocation == null) {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ storeLocation = storeLocation + '/' + System.nanoTime() + '/' + theSplit.index
CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
+ LOGGER.info("Temp storeLocation taken is " + storeLocation)
+ var mergeStatus = false
+ var mergeNumber = ""
+ try {
+ var dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
- // sorting the table block info List.
- var tableBlockInfoList = carbonSparkPartition.tableBlockInfos
+ // sorting the table block info List.
+ var tableBlockInfoList = carbonSparkPartition.tableBlockInfos
- Collections.sort(tableBlockInfoList)
+ Collections.sort(tableBlockInfoList)
- val segmentMapping: java.util.Map[String, TaskBlockInfo] =
- CarbonCompactionUtil.createMappingForSegments(tableBlockInfoList)
+ val segmentMapping: java.util.Map[String, TaskBlockInfo] =
+ CarbonCompactionUtil.createMappingForSegments(tableBlockInfoList)
- val dataFileMetadataSegMapping: java.util.Map[String, List[DataFileFooter]] =
- CarbonCompactionUtil.createDataFileFooterMappingForSegments(tableBlockInfoList)
+ val dataFileMetadataSegMapping: java.util.Map[String, List[DataFileFooter]] =
+ CarbonCompactionUtil.createDataFileFooterMappingForSegments(tableBlockInfoList)
- carbonLoadModel.setStorePath(hdfsStoreLocation)
+ carbonLoadModel.setStorePath(hdfsStoreLocation)
- // taking the last table block info for getting the segment properties.
- val listMetadata = dataFileMetadataSegMapping.get(tableBlockInfoList.get
- (tableBlockInfoList.size()-1).getSegmentId())
+ // taking the last table block info for getting the segment properties.
+ val listMetadata = dataFileMetadataSegMapping.get(tableBlockInfoList.get
+ (tableBlockInfoList.size() - 1).getSegmentId()
+ )
- val colCardinality: Array[Int] = listMetadata.get(listMetadata.size() - 1).getSegmentInfo
- .getColumnCardinality
+ val colCardinality: Array[Int] = listMetadata.get(listMetadata.size() - 1).getSegmentInfo
+ .getColumnCardinality
- val segmentProperties = new SegmentProperties(
- listMetadata.get(listMetadata.size() - 1).getColumnInTable,
- colCardinality
- )
+ val segmentProperties = new SegmentProperties(
+ listMetadata.get(listMetadata.size() - 1).getColumnInTable,
+ colCardinality
+ )
- val exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties, databaseName,
- factTableName, hdfsStoreLocation, carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable,
- dataFileMetadataSegMapping
- )
- val exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties, schemaName,
++ val exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties, databaseName,
+ factTableName, hdfsStoreLocation, carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable,
+ dataFileMetadataSegMapping
+ )
- // fire a query and get the results.
- var result2: util.List[RawResultIterator] = null
- try {
- result2 = exec.processTableBlocks()
- } catch {
- case e: Throwable =>
- exec.clearDictionaryFromQueryModel
- LOGGER.error(e)
- if (null != e.getMessage) {
- sys.error("Exception occurred in query execution :: " + e.getMessage)
- } else {
- sys.error("Exception occurred in query execution.Please check logs.")
- }
- }
+ // fire a query and get the results.
+ var result2: util.List[RawResultIterator] = null
+ try {
+ result2 = exec.processTableBlocks()
+ } catch {
+ case e: Throwable =>
+ exec.clearDictionaryFromQueryModel
+ LOGGER.error(e)
+ if (null != e.getMessage) {
+ sys.error("Exception occurred in query execution :: " + e.getMessage)
+ } else {
+ sys.error("Exception occurred in query execution.Please check logs.")
+ }
+ }
- val mergeNumber = mergedLoadName
- .substring(mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
- CarbonCommonConstants.LOAD_FOLDER.length(), mergedLoadName.length()
+ mergeNumber = mergedLoadName
+ .substring(mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
+ CarbonCommonConstants.LOAD_FOLDER.length(), mergedLoadName.length()
+ )
+
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(schemaName,
++ val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
+ factTableName,
+ carbonLoadModel.getTaskNo,
+ "0",
+ mergeNumber,
+ true
)
- val tempStoreLoc = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName,
- factTableName,
- carbonLoadModel.getTaskNo,
- "0",
- mergeNumber
- )
+ carbonLoadModel.setSegmentId(mergeNumber)
+ carbonLoadModel.setPartitionId("0")
+ val merger =
+ new RowResultMerger(result2,
- schemaName,
++ databaseName,
+ factTableName,
+ segmentProperties,
+ tempStoreLoc,
+ carbonLoadModel,
+ colCardinality
+ )
+ mergeStatus = merger.mergerSlice()
- carbonLoadModel.setSegmentId(mergeNumber)
- carbonLoadModel.setPartitionId("0")
- val merger =
- new RowResultMerger(result2,
- databaseName,
- factTableName,
- segmentProperties,
- tempStoreLoc,
- carbonLoadModel,
- colCardinality
- )
- val mergeStatus = merger.mergerSlice()
+ }
+ catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ throw e
+ }
+ finally {
+ // delete temp location data
+ val newSlice = CarbonCommonConstants.LOAD_FOLDER + mergeNumber
+ try {
+ val isCompactionFlow = true
+ CarbonLoaderUtil
+ .deleteLocalDataLoadFolderLocation(carbonLoadModel, newSlice, isCompactionFlow)
+ } catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ }
+ }
var finished = false
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
index 00bf5b2,0000000..5a1dedc
mode 100644,000000..100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
@@@ -1,253 -1,0 +1,290 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.carbondata.spark.rdd
+
+import java.util
+
+import scala.collection.JavaConverters._
+import scala.reflect.ClassTag
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.mapreduce.Job
+import org.apache.spark.{Logging, Partition, SparkContext, TaskContext}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.hive.DistributionUtil
+
+import org.carbondata.common.CarbonIterator
+import org.carbondata.common.logging.LogServiceFactory
+import org.carbondata.core.cache.dictionary.Dictionary
+import org.carbondata.core.carbon.datastore.block.{Distributable, TableBlockInfo}
+import org.carbondata.core.carbon.querystatistics.{QueryStatistic, QueryStatisticsRecorder}
+import org.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit}
+import org.carbondata.scan.executor.QueryExecutorFactory
+import org.carbondata.scan.expression.Expression
+import org.carbondata.scan.model.QueryModel
+import org.carbondata.scan.result.BatchResult
+import org.carbondata.scan.result.iterator.ChunkRowIterator
+import org.carbondata.spark.RawValue
+import org.carbondata.spark.load.CarbonLoaderUtil
+import org.carbondata.spark.util.QueryPlanUtil
+
+class CarbonSparkPartition(rddId: Int, val idx: Int,
+ val locations: Array[String],
+ val tableBlockInfos: util.List[TableBlockInfo])
+ extends Partition {
+
+ override val index: Int = idx
+
+ // val serializableHadoopSplit = new SerializableWritable[Array[String]](locations)
+ override def hashCode(): Int = {
+ 41 * (41 + rddId) + idx
+ }
+}
+
+ /**
+ * This RDD is used to perform query on CarbonData file. Before sending tasks to scan
+ * CarbonData file, this RDD will leverage CarbonData's index information to do CarbonData file
+ * level filtering in driver side.
+ */
+class CarbonScanRDD[V: ClassTag](
+ sc: SparkContext,
+ queryModel: QueryModel,
+ filterExpression: Expression,
+ keyClass: RawValue[V],
+ @transient conf: Configuration,
+ tableCreationTime: Long,
+ schemaLastUpdatedTime: Long,
+ baseStoreLocation: String)
+ extends RDD[V](sc, Nil) with Logging {
+
+ val defaultParallelism = sc.defaultParallelism
+
+ override def getPartitions: Array[Partition] = {
+ val statisticRecorder = new QueryStatisticsRecorder(queryModel.getQueryId)
+ val startTime = System.currentTimeMillis()
+ val (carbonInputFormat: CarbonInputFormat[Array[Object]], job: Job) =
+ QueryPlanUtil.createCarbonInputFormat(queryModel.getAbsoluteTableIdentifier)
+
+ val result = new util.ArrayList[Partition](defaultParallelism)
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ // set filter resolver tree
+ try {
+ // before applying filter check whether segments are available in the table.
+ val splits = carbonInputFormat.getSplits(job)
+ if (!splits.isEmpty) {
- var filterResolver = carbonInputFormat
++ val filterResolver = carbonInputFormat
+ .getResolvedFilter(job.getConfiguration, filterExpression)
+ CarbonInputFormat.setFilterPredicates(job.getConfiguration, filterResolver)
+ queryModel.setFilterExpressionResolverTree(filterResolver)
+ }
+ }
+ catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ sys.error("Exception occurred in query execution :: " + e.getMessage)
+ }
+ // get splits
+ val splits = carbonInputFormat.getSplits(job)
+ if (!splits.isEmpty) {
+ val carbonInputSplits = splits.asScala.map(_.asInstanceOf[CarbonInputSplit])
+
+ val blockList = carbonInputSplits.map(inputSplit =>
+ new TableBlockInfo(inputSplit.getPath.toString,
+ inputSplit.getStart, inputSplit.getSegmentId,
+ inputSplit.getLocations, inputSplit.getLength
+ ).asInstanceOf[Distributable]
+ )
+ if (blockList.nonEmpty) {
+ // group blocks to nodes, tasks
+ val startTime = System.currentTimeMillis
+ var statistic = new QueryStatistic
+ val activeNodes = DistributionUtil
+ .ensureExecutorsAndGetNodeList(blockList.toArray, sparkContext)
+ val nodeBlockMapping =
+ CarbonLoaderUtil.nodeBlockTaskMapping(blockList.asJava, -1, defaultParallelism,
+ activeNodes.toList.asJava
+ )
+ val timeElapsed: Long = System.currentTimeMillis - startTime
+ statistic.addStatistics("Total Time taken in block(s) allocation", System.currentTimeMillis)
+ statisticRecorder.recordStatistics(statistic);
+ statistic = new QueryStatistic
+ var i = 0
+ // Create Spark Partition for each task and assign blocks
+ nodeBlockMapping.asScala.foreach { entry =>
+ entry._2.asScala.foreach { blocksPerTask => {
+ val tableBlockInfo = blocksPerTask.asScala.map(_.asInstanceOf[TableBlockInfo])
+ if (blocksPerTask.size() != 0) {
+ result
+ .add(new CarbonSparkPartition(id, i, Seq(entry._1).toArray, tableBlockInfo.asJava))
+ i += 1
+ }
+ }
+ }
+ }
+ val noOfBlocks = blockList.size
+ val noOfNodes = nodeBlockMapping.size
+ val noOfTasks = result.size()
+ logInfo(s"Identified no.of.Blocks: $noOfBlocks,"
+ + s"parallelism: $defaultParallelism , " +
+ s"no.of.nodes: $noOfNodes, no.of.tasks: $noOfTasks"
+ )
+ statistic.addStatistics("Time taken to identify Block(s) to scan", System.currentTimeMillis)
+ statisticRecorder.recordStatistics(statistic);
+ statisticRecorder.logStatistics
+ result.asScala.foreach { r =>
+ val cp = r.asInstanceOf[CarbonSparkPartition]
+ logInfo(s"Node : " + cp.locations.toSeq.mkString(",")
+ + ", No.Of Blocks : " + cp.tableBlockInfos.size()
+ )
+ }
+ } else {
+ logInfo("No blocks identified to scan")
+ val nodesPerBlock = new util.ArrayList[TableBlockInfo]()
+ result.add(new CarbonSparkPartition(id, 0, Seq("").toArray, nodesPerBlock))
+ }
+ }
+ else {
+ logInfo("No valid segments found to scan")
+ val nodesPerBlock = new util.ArrayList[TableBlockInfo]()
+ result.add(new CarbonSparkPartition(id, 0, Seq("").toArray, nodesPerBlock))
+ }
+ result.toArray(new Array[Partition](result.size()))
+ }
+
+ override def compute(thepartition: Partition, context: TaskContext): Iterator[V] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
+ val iter = new Iterator[V] {
+ var rowIterator: CarbonIterator[Array[Any]] = _
+ var queryStartTime: Long = 0
+ try {
+ val carbonSparkPartition = thepartition.asInstanceOf[CarbonSparkPartition]
+ if(!carbonSparkPartition.tableBlockInfos.isEmpty) {
+ queryModel.setQueryId(queryModel.getQueryId + "_" + carbonSparkPartition.idx)
+ // fill table block info
+ queryModel.setTableBlockInfos(carbonSparkPartition.tableBlockInfos)
+ queryStartTime = System.currentTimeMillis
+
+ val carbonPropertiesFilePath = System.getProperty("carbon.properties.filepath", null)
+ logInfo("*************************" + carbonPropertiesFilePath)
+ if (null == carbonPropertiesFilePath) {
+ System.setProperty("carbon.properties.filepath",
+ System.getProperty("user.dir") + '/' + "conf" + '/' + "carbon.properties")
+ }
+ // execute query
+ rowIterator = new ChunkRowIterator(
+ QueryExecutorFactory.getQueryExecutor(queryModel).execute(queryModel).
+ asInstanceOf[CarbonIterator[BatchResult]]).asInstanceOf[CarbonIterator[Array[Any]]]
+
+ }
+ } catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ if (null != e.getMessage) {
+ sys.error("Exception occurred in query execution :: " + e.getMessage)
+ } else {
+ sys.error("Exception occurred in query execution.Please check logs.")
+ }
+ }
+
+ var havePair = false
+ var finished = false
+ var recordCount = 0
+
+ override def hasNext: Boolean = {
+ if (!finished && !havePair) {
+ finished = (null == rowIterator) || (!rowIterator.hasNext)
+ havePair = !finished
+ }
+ if (finished) {
+ clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
++ if (null != queryModel.getStatisticsRecorder) {
++ val queryStatistic = new QueryStatistic
++ queryStatistic
++ .addStatistics("Total Time taken to execute the query in executor Side",
++ System.currentTimeMillis - queryStartTime
++ )
++ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
+ queryModel.getStatisticsRecorder.logStatistics();
+ }
+ }
+ !finished
+ }
+
+ override def next(): V = {
+ if (!hasNext) {
+ throw new java.util.NoSuchElementException("End of stream")
+ }
+ havePair = false
+ recordCount += 1
+ if (queryModel.getLimit != -1 && recordCount >= queryModel.getLimit) {
+ clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
++ if (null != queryModel.getStatisticsRecorder) {
++ val queryStatistic = new QueryStatistic
++ queryStatistic
++ .addStatistics("Total Time taken to execute the query in executor Side",
++ System.currentTimeMillis - queryStartTime
++ )
++ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
+ queryModel.getStatisticsRecorder.logStatistics();
+ }
+ }
+ keyClass.getValue(rowIterator.next())
+ }
+ def clearDictionaryCache(columnToDictionaryMap: java.util.Map[String, Dictionary]) = {
+ if (null != columnToDictionaryMap) {
+ org.carbondata.spark.util.CarbonQueryUtil
+ .clearColumnDictionaryCache(columnToDictionaryMap)
+ }
+ }
+ }
+ iter
+ }
+
+ /**
+ * Get the preferred locations where to launch this task.
+ */
- override def getPreferredLocations(partition: Partition): Seq[String] = {
- val theSplit = partition.asInstanceOf[CarbonSparkPartition]
- theSplit.locations.filter(_ != "localhost")
- }
++ override def getPreferredLocations(partition: Partition): Seq[String] = {
++ val theSplit = partition.asInstanceOf[CarbonSparkPartition]
++ val firstOptionLocation = theSplit.locations.filter(_ != "localhost")
++ val tableBlocks = theSplit.tableBlockInfos
++ // node name and count mapping
++ val blockMap = new util.LinkedHashMap[String, Integer]()
++
++ tableBlocks.asScala.foreach(tableBlock => tableBlock.getLocations.foreach(
++ location => {
++ if (!firstOptionLocation.exists(location.equalsIgnoreCase(_))) {
++ val currentCount = blockMap.get(location)
++ if (currentCount == null) {
++ blockMap.put(location, 1)
++ } else {
++ blockMap.put(location, currentCount + 1)
++ }
++ }
++ }
++ )
++ )
++
++ val sortedList = blockMap.entrySet().asScala.toSeq.sortWith((nodeCount1, nodeCount2) => {
++ nodeCount1.getValue > nodeCount2.getValue
++ }
++ )
++
++ val sortedNodesList = sortedList.map(nodeCount => nodeCount.getKey).take(2)
++ firstOptionLocation ++ sortedNodesList
++ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 1693008,b3effd3..5406e77
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@@ -79,22 -93,40 +79,35 @@@ object CarbonScalaUtil
}
}
- def convertCarbonToSparkDataType(dataType: DataType): types.DataType = {
- dataType match {
- case DataType.STRING => StringType
- case DataType.INT => IntegerType
- case DataType.LONG => LongType
- case DataType.DOUBLE => DoubleType
- case DataType.BOOLEAN => BooleanType
- case DataType.DECIMAL => DecimalType.SYSTEM_DEFAULT
- case DataType.TIMESTAMP => TimestampType
+ def getKettleHomePath(sqlContext: SQLContext): String = {
+ val carbonHome = System.getenv("CARBON_HOME")
+ var kettleHomePath: String = null
+ if (carbonHome != null) {
+ kettleHomePath = System.getenv("CARBON_HOME") + "/processing/carbonplugins"
+ }
+ if (kettleHomePath == null) {
+ kettleHomePath = sqlContext.getConf("carbon.kettle.home", null)
}
+ if (null == kettleHomePath) {
+ kettleHomePath = CarbonProperties.getInstance.getProperty("carbon.kettle.home")
+ }
+ kettleHomePath
}
+ def updateDataType(
+ currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = {
+ currentDataType match {
+ case decimal: DecimalType =>
+ val scale = currentDataType.asInstanceOf[DecimalType].scale
+ DecimalType(DecimalType.MAX_PRECISION, scale)
+ case _ =>
+ currentDataType
+ }
+ }
+
+ case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
+
object CarbonSparkUtil {
- def createBaseRDD(carbonContext: CarbonContext, carbonTable: CarbonTable): TransformHolder = {
- val relation = CarbonEnv.getInstance(carbonContext).carbonCatalog
- .lookupRelation1(Option(carbonTable.getDatabaseName),
- carbonTable.getFactTableName, None)(carbonContext).asInstanceOf[CarbonRelation]
- val rdd = new SchemaRDD(carbonContext, relation)
- rdd.registerTempTable(carbonTable.getFactTableName)
- TransformHolder(rdd, createSparkMeta(carbonTable))
- }
def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
val dimensionsAttr = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index b381079,0e333dd..780c022
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@@ -80,11 -92,8 +92,9 @@@ class TestLoadDataWithDictionaryExclude
}
test("test load data with dictionary exclude & include and with empty dimension") {
+ sql("select ID from t3").show()
checkAnswer(
- sql("select ID from t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6), Row(7),
- Row(8), Row(9), Row(10), Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row
- (17), Row(18), Row(19), Row(20))
+ sql("select ID from exclude_include_t3"), sql("select ID from exclude_include_hive_t3")
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index 483a766,db67c86..a29cefb
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@@ -60,45 -61,82 +60,82 @@@ class TimestampDataTypeDirectDictionary
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
- sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryCube OPTIONS" +
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
-
} catch {
case x: Throwable => CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
}
}
- test("select doj from directDictionaryTable") {
+ test("test direct dictionary for not null condition") {
checkAnswer(
- sql("select doj from directDictionaryTable"),
- sql("select doj from directDictionaryCube where doj is not null"),
++ sql("select doj from directDictionaryTable where doj is not null"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
)
)
}
+ test("test direct dictionary for getting all the values") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+ Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+ Row(null)
+ )
+ )
+ }
+
+ test("test direct dictionary for not equals condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj != '2016-04-14 15:00:09.0'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+ )
+ )
+ }
+
+ test("test direct dictionary for null condition") {
+ checkAnswer(
+ sql("select doj from directDictionaryCube where doj is null"),
+ Seq(Row(null)
+ )
+ )
+ }
- test("select doj from directDictionaryCube with equals filter") {
+ test("select doj from directDictionaryTable with equals filter") {
checkAnswer(
- sql("select doj from directDictionaryCube where doj='2016-03-14 15:00:09'"),
+ sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09")))
)
}
- test("select doj from directDictionaryTable with greater than filter") {
- test("select doj from directDictionaryCube with regexp_replace equals filter") {
++ test("select doj from directDictionaryTable with regexp_replace equals filter") {
+ checkAnswer(
- sql("select doj from directDictionaryCube where regexp_replace(doj, '-', '/') = '2016/03/14 15:00:09'"),
++ sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') = '2016/03/14 15:00:09'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09")))
+ )
+ }
-
- test("select doj from directDictionaryCube with regexp_replace NOT IN filter") {
++
++ test("select doj from directDictionaryTable with regexp_replace NOT IN filter") {
+ checkAnswer(
- sql("select doj from directDictionaryCube where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')"),
++ sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')"),
+ Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")), Row(null))
+ )
+ }
-
- test("select doj from directDictionaryCube with greater than filter") {
++
++ test("select doj from directDictionaryTable with greater than filter") {
checkAnswer(
- sql("select doj from directDictionaryCube where doj>'2016-03-14 15:00:09'"),
+ sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"),
Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")))
)
-
}
- test("select count(doj) from directDictionaryCube") {
+ test("select count(doj) from directDictionaryTable") {
checkAnswer(
- sql("select count(doj) from directDictionaryCube"),
+ sql("select count(doj) from directDictionaryTable"),
Seq(Row(2))
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
----------------------------------------------------------------------
diff --cc integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
index 161a19e,7cb6dfd..4075e60
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
@@@ -41,14 -41,26 +41,26 @@@ class AllDataTypesTestCaseFilter extend
}
- test("select empno,empname,utilization,count(salary),sum(empno) from alldatatypescubeFilter where empname in ('arvind','ayushi') group by empno,empname,utilization") {
+ test("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestableFilter where empname in ('arvind','ayushi') group by empno,empname,utilization") {
checkAnswer(
- sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypescubeFilter where empname in ('arvind','ayushi') group by empno,empname,utilization"),
- sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypescubeFilter_hive where empname in ('arvind','ayushi') group by empno,empname,utilization"))
+ sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestableFilter where empname in ('arvind','ayushi') group by empno,empname,utilization"),
+ sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestableFilter_hive where empname in ('arvind','ayushi') group by empno,empname,utilization"))
}
-
+
+ test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')") {
+ checkAnswer(
+ sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"),
+ sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"))
+ }
+
+ test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'") {
+ checkAnswer(
+ sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"),
+ sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"))
+ }
+
override def afterAll {
- sql("drop table alldatatypescubeFilter")
- sql("drop table alldatatypescubeFilter_hive")
+ sql("drop table alldatatypestableFilter")
+ sql("drop table alldatatypestableFilter_hive")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/lcm/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/lcm/locks/CarbonLockFactory.java
index 10aa1fd,0000000..0fae687
mode 100644,000000..100644
--- a/processing/src/main/java/org/carbondata/lcm/locks/CarbonLockFactory.java
+++ b/processing/src/main/java/org/carbondata/lcm/locks/CarbonLockFactory.java
@@@ -1,73 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.lcm.locks;
+
+import org.carbondata.core.carbon.CarbonTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.util.CarbonProperties;
+
+/**
+ * This class is a Lock factory class which is used to provide lock objects.
+ * Using this lock object client can request the lock and unlock.
+ */
+public class CarbonLockFactory {
+
+ /**
+ * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
+ */
+ private static String lockTypeConfigured;
+
+ static {
- CarbonLockFactory.updateZooKeeperLockingStatus();
++ CarbonLockFactory.getLockTypeConfigured();
+ }
+
+ /**
+ * This method will determine the lock type.
+ *
+ * @param tableIdentifier
+ * @param lockFile
+ * @return
+ */
+ public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier,
+ String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(tableIdentifier, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
++ *
++ * @param locFileLocation
++ * @param lockFile
++ * @return carbon lock
++ */
++ public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
++ switch (lockTypeConfigured.toUpperCase()) {
++ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
++ return new LocalFileLock(locFileLocation, lockFile);
++
++ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
++ return new ZooKeeperLocking(locFileLocation, lockFile);
++
++ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
++ return new HdfsFileLock(locFileLocation, lockFile);
++
++ default:
++ throw new UnsupportedOperationException("Not supported the lock type");
++ }
++ }
++
++ /**
+ * This method will set the zookeeper status whether zookeeper to be used for locking or not.
+ */
- private static void updateZooKeeperLockingStatus() {
++ private static void getLockTypeConfigured() {
+ lockTypeConfigured = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
-
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/lcm/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/lcm/locks/HdfsFileLock.java
index b45ae96,0000000..3305477
mode 100644,000000..100644
--- a/processing/src/main/java/org/carbondata/lcm/locks/HdfsFileLock.java
+++ b/processing/src/main/java/org/carbondata/lcm/locks/HdfsFileLock.java
@@@ -1,91 -1,0 +1,106 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.lcm.locks;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
++import org.carbondata.common.logging.LogService;
++import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.datastorage.store.impl.FileFactory;
++import org.carbondata.core.util.CarbonProperties;
+
+/**
+ * This class is used to handle the HDFS File locking.
+ * This is acheived using the concept of acquiring the data out stream using Append option.
+ */
+public class HdfsFileLock extends AbstractCarbonLock {
+
++ private static final LogService LOGGER =
++ LogServiceFactory.getLogService(HdfsFileLock.class.getName());
+ /**
+ * location hdfs file location
+ */
+ private String location;
+
+ private DataOutputStream dataOutputStream;
+
+ public static String tmpPath;
+
+ static {
- tmpPath = System.getProperty("hadoop.tmp.dir");
++ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
++ System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
++ }
++
++ /**
++ * @param lockFileLocation
++ * @param lockFile
++ */
++ public HdfsFileLock(String lockFileLocation, String lockFile) {
++ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
++ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
++ LOGGER.info("HDFS lock path:"+this.location);
++ initRetry();
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName()
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
++ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
++ .getTableName(), lockFile);
+ initRetry();
+ }
+
+ /* (non-Javadoc)
+ * @see org.carbondata.core.locks.ICarbonLock#lock()
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
+ }
+ dataOutputStream =
+ FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
+
+ return true;
+
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.carbondata.core.locks.ICarbonLock#unlock()
+ */
+ @Override public boolean unlock() {
+ if (null != dataOutputStream) {
+ try {
+ dataOutputStream.close();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/lcm/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/lcm/locks/LocalFileLock.java
index 88f9a23,0000000..15374a3
mode 100644,000000..100644
--- a/processing/src/main/java/org/carbondata/lcm/locks/LocalFileLock.java
+++ b/processing/src/main/java/org/carbondata/lcm/locks/LocalFileLock.java
@@@ -1,151 -1,0 +1,159 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.lcm.locks;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.datastorage.store.impl.FileFactory;
+
+/**
+ * This class handles the file locking in the local file system.
+ * This will be handled using the file channel lock API.
+ */
+public class LocalFileLock extends AbstractCarbonLock {
+ /**
+ * location is the location of the lock file.
+ */
+ private String location;
+
+ /**
+ * fileOutputStream of the local lock file
+ */
+ private FileOutputStream fileOutputStream;
+
+ /**
+ * channel is the FileChannel of the lock file.
+ */
+ private FileChannel channel;
+
+ /**
+ * fileLock NIO FileLock Object
+ */
+ private FileLock fileLock;
+
+ /**
+ * lock file
+ */
+ private String lockFile;
+
+ public static final String tmpPath;
+
+ private String tableName;
+
+ private String databaseName;
+
+ /**
+ * LOGGER for logging the messages.
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LocalFileLock.class.getName());
+
+ static {
+ tmpPath = System.getProperty("java.io.tmpdir");
+ }
+
+ /**
++ * @param lockFileLocation
++ * @param lockFile
++ */
++ public LocalFileLock(String lockFileLocation, String lockFile) {
++ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
++ this.lockFile = lockFile;
++ initRetry();
++ }
++
++ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName();
- this.lockFile = lockFile;
++ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
++ .getTableName(), lockFile);
+ initRetry();
+ }
+
+ /**
+ * Lock API for locking of the file channel of the lock file.
+ *
+ * @return
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) {
+ FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath));
+ }
+ String lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR +
+ lockFile;
+ if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
+ }
+
+ fileOutputStream = new FileOutputStream(lockFilePath);
+ channel = fileOutputStream.getChannel();
+ try {
+ fileLock = channel.tryLock();
+ } catch (OverlappingFileLockException e) {
+ return false;
+ }
+ if (null != fileLock) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ return false;
+ }
+
+ }
+
+ /**
+ * Unlock API for unlocking of the acquired lock.
+ *
+ * @return
+ */
+ @Override public boolean unlock() {
+ boolean status;
+ try {
+ if (null != fileLock) {
+ fileLock.release();
+ }
+ status = true;
+ } catch (IOException e) {
+ status = false;
+ } finally {
+ if (null != fileOutputStream) {
+ try {
+ fileOutputStream.close();
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+ }
+ return status;
+ }
+
+}
[34/47] incubator-carbondata git commit: [CARBONDATA-120] Explain
extended command for carbon commands (#886)
Posted by ra...@apache.org.
[CARBONDATA-120] Explain extended command for carbon commands (#886)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b3779e46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b3779e46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b3779e46
Branch: refs/heads/master
Commit: b3779e46e43b432613922101d05ab5e21eef40c0
Parents: 935e0d3
Author: nareshpr <pr...@gmail.com>
Authored: Fri Jul 29 00:40:22 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jul 29 00:40:22 2016 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/CarbonSqlParser.scala | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b3779e46/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 0fd841d..5c25406 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.{SqlLexical, _}
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.execution.ExplainCommand
import org.apache.spark.sql.execution.command.{DimensionRelation, _}
import org.apache.spark.sql.execution.datasources.DescribeCommand
import org.apache.spark.sql.hive.HiveQlWrapper
@@ -78,6 +79,7 @@ class CarbonSqlParser()
protected val DROP = carbonKeyWord("DROP")
protected val ESCAPECHAR = carbonKeyWord("ESCAPECHAR")
protected val EXCLUDE = carbonKeyWord("EXCLUDE")
+ protected val EXPLAIN = carbonKeyWord("EXPLAIN")
protected val EXTENDED = carbonKeyWord("EXTENDED")
protected val FORMATTED = carbonKeyWord("FORMATTED")
protected val FACT = carbonKeyWord("FACT")
@@ -199,7 +201,9 @@ class CarbonSqlParser()
private def carbonKeyWord(keys: String) =
("(?i)" + keys).r
- override protected lazy val start: Parser[LogicalPlan] =
+ override protected lazy val start: Parser[LogicalPlan] = explainPlan | startCommand
+
+ protected lazy val startCommand: Parser[LogicalPlan] =
loadManagement | describeTable | showLoads | alterTable | createTable
protected lazy val loadManagement: Parser[LogicalPlan] = deleteLoadsByID | deleteLoadsByLoadDate |
@@ -1342,4 +1346,13 @@ class CarbonSqlParser()
case schemaName ~ cubeName => CleanFiles(schemaName, cubeName.toLowerCase())
}
+ protected lazy val explainPlan: Parser[LogicalPlan] =
+ (EXPLAIN ~> opt(EXTENDED)) ~ startCommand ^^ {
+ case isExtended ~ logicalPlan =>
+ logicalPlan match {
+ case plan: CreateTable => ExplainCommand(logicalPlan, extended = isExtended.isDefined)
+ case _ => ExplainCommand(OneRowRelation)
+ }
+ }
+
}
[07/47] incubator-carbondata git commit: Struct of array is failing
because of complex dimension ordinal (#848)
Posted by ra...@apache.org.
Struct of array is failing because of complex dimension ordinal (#848)
Problem:
Assignment of complex dimension ordinal was not correct as it was not increment and two children was getting same complex dimension ordinal
Second issue was selecting all the block indexes of complex dimension children.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/07116d24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/07116d24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/07116d24
Branch: refs/heads/master
Commit: 07116d242db652ad8edf6e80526f6a377295343d
Parents: 4cff504
Author: Kumar Vishal <ku...@gmail.com>
Authored: Fri Jul 22 21:13:31 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Fri Jul 22 21:13:31 2016 +0530
----------------------------------------------------------------------
.../metadata/schema/table/CarbonTable.java | 48 ++++++++++++++------
.../schema/table/column/CarbonDimension.java | 4 ++
.../query/carbon/executor/util/QueryUtil.java | 34 +++++++-------
.../spark/src/test/resources/struct_all.csv | 4 ++
.../src/test/resources/structusingstruct.csv | 2 +
.../complexType/TestComplexTypeQuery.scala | 33 +++++++++++++-
6 files changed, 91 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index 6023e7a..4d16659 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -121,23 +121,20 @@ public class CarbonTable implements Serializable {
int columnGroupOrdinal = -1;
int previousColumnGroupId = -1;
List<ColumnSchema> listOfColumns = tableSchema.getListOfColumns();
- int complexTypeOrdinal = 0;
- ;
+ int complexTypeOrdinal = -1;
for (int i = 0; i < listOfColumns.size(); i++) {
ColumnSchema columnSchema = listOfColumns.get(i);
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, complexTypeOrdinal++);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
dimensions.add(complexDimension);
- int lastDimensionOrdianl=dimensionOrdinal;
dimensionOrdinal =
readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
- listOfColumns, complexDimension, complexTypeOrdinal);
+ listOfColumns, complexDimension);
i = dimensionOrdinal - 1;
- int incrementInOrdinal=dimensionOrdinal-lastDimensionOrdianl;
- complexTypeOrdinal+=incrementInOrdinal;
+ complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
@@ -171,24 +168,21 @@ public class CarbonTable implements Serializable {
* @return
*/
private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount,
- List<ColumnSchema> listOfColumns, CarbonDimension parentDimension,
- int complexDimensionOrdianl) {
+ List<ColumnSchema> listOfColumns, CarbonDimension parentDimension) {
for (int i = 0; i < childCount; i++) {
ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal);
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdianl++);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(),
- listOfColumns, complexDimension, complexDimensionOrdianl);
+ listOfColumns, complexDimension);
} else {
- parentDimension.getListOfChildDimensions().add(
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1,
- complexDimensionOrdianl++));
+ parentDimension.getListOfChildDimensions()
+ .add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
}
}
}
@@ -196,6 +190,30 @@ public class CarbonTable implements Serializable {
}
/**
+ * Read all primitive/complex children and set it as list of child carbon dimension to parent
+ * dimension
+ *
+ * @param dimensionOrdinal
+ * @param childCount
+ * @param listOfColumns
+ * @param parentDimension
+ * @return
+ */
+ private int assignComplexOrdinal(CarbonDimension parentDimension, int complexDimensionOrdianl) {
+ for (int i = 0; i < parentDimension.getNumberOfChild(); i++) {
+ CarbonDimension dimension = parentDimension.getListOfChildDimensions().get(i);
+ if (dimension.getNumberOfChild() > 0) {
+ dimension.setComplexTypeOridnal(++complexDimensionOrdianl);
+ complexDimensionOrdianl = assignComplexOrdinal(dimension, complexDimensionOrdianl);
+ } else {
+ parentDimension.getListOfChildDimensions().get(i)
+ .setComplexTypeOridnal(++complexDimensionOrdianl);
+ }
+ }
+ return complexDimensionOrdianl;
+ }
+
+ /**
* @return the databaseName
*/
public String getDatabaseName() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
index 3a4dd11..d3df181 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
@@ -114,6 +114,10 @@ public class CarbonDimension extends CarbonColumn {
return complexTypeOrdinal;
}
+ public void setComplexTypeOridnal(int complexTypeOrdinal) {
+ this.complexTypeOrdinal = complexTypeOrdinal;
+ }
+
/**
* to generate the hash code for this class
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index 241f5b4..2433cbc 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -252,19 +252,22 @@ public class QueryUtil {
blockIndex =
dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ }
}
for (int i = 0; i < dimAggInfo.size(); i++) {
blockIndex = dimensionOrdinalToBlockMapping.get(dimAggInfo.get(i).getDim().getOrdinal());
+ // not adding the children dimension as dimension aggregation
+ // is not push down in case of complex dimension
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, dimAggInfo.get(i).getDim());
-
}
for (int i = 0; i < customAggregationDimension.size(); i++) {
blockIndex =
dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+ // not adding the children dimension as dimension aggregation
+ // is not push down in case of complex dimension
dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, customAggregationDimension.get(i));
}
return ArrayUtils
.toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
@@ -274,16 +277,13 @@ public class QueryUtil {
* Below method will be used to add the children block index
* this will be basically for complex dimension which will have children
*
- * @param startBlockIndex start block index
- * @param blockIndexList block index list
- * @param dimension parent dimension
+ * @param blockIndexes block indexes
+ * @param dimension parent dimension
*/
- private static void addChildrenBlockIndex(int startBlockIndex, Set<Integer> blockIndexList,
- CarbonDimension dimension) {
+ private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) {
for (int i = 0; i < dimension.numberOfChild(); i++) {
- blockIndexList.add(++startBlockIndex);
- addChildrenBlockIndex(startBlockIndex, blockIndexList,
- dimension.getListOfChildDimensions().get(i));
+ addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
+ blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
}
}
@@ -379,7 +379,7 @@ public class QueryUtil {
if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
dictionaryDimensionFromQuery);
- } else if(!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
dictionaryDimensionFromQuery
.add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
}
@@ -873,7 +873,7 @@ public class QueryUtil {
&& queryDimension.getDimension().numberOfChild() == 0) {
dictionaryDimensionBlockIndex
.add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- } else if(queryDimension.getDimension().numberOfChild() == 0){
+ } else if (queryDimension.getDimension().numberOfChild() == 0) {
noDictionaryDimensionBlockIndex
.add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
}
@@ -1078,9 +1078,9 @@ public class QueryUtil {
dimension.getColName(), ++parentBlockIndex));
break;
default:
- boolean isDirectDictionary = CarbonUtil.hasEncoding(
- dimension.getListOfChildDimensions().get(i).getEncoder(),
- Encoding.DIRECT_DICTIONARY);
+ boolean isDirectDictionary = CarbonUtil
+ .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
+ Encoding.DIRECT_DICTIONARY);
parentQueryType.addChildren(
new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
dimension.getColName(), ++parentBlockIndex,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/struct_all.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/struct_all.csv b/integration/spark/src/test/resources/struct_all.csv
new file mode 100644
index 0000000..02b6387
--- /dev/null
+++ b/integration/spark/src/test/resources/struct_all.csv
@@ -0,0 +1,4 @@
+10&10$10&10
+20&20$20&20
+30&30$30&30
+40&40$40&40
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/resources/structusingstruct.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/structusingstruct.csv b/integration/spark/src/test/resources/structusingstruct.csv
new file mode 100644
index 0000000..5a2bae1
--- /dev/null
+++ b/integration/spark/src/test/resources/structusingstruct.csv
@@ -0,0 +1,2 @@
+def$klm&abc$12
+pri$sac&pra$18
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/07116d24/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index d17840f..ab48133 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -37,12 +37,37 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
sql("drop table if exists complexcarbontable").show
sql("drop table if exists complexhivetable").show
+ sql("drop table if exists structusingstructCarbon").show
+ sql("drop table if exists structusingstructHive").show
+ sql("drop table if exists structusingarraycarbon").show
+ sql("drop table if exists structusingarrayhive").show
sql("create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId double,contractNumber double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId', 'DICTIONARY_EXCLUDE'='channelsId','COLUMN_GROUP'='(ROMSize,ROMName)')");
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexcarbontable OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', 'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
sql("create table complexhivetable(deviceInformationId int, channelsId string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId double,contractNumber double)row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by ':'")
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexhivetable");
+
+ sql("create table structusingarraycarbon (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>) STORED BY 'org.apache.carbondata.format'");
+ sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarraycarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+ sql("create table structusingarrayhive (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>)row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by '&'");
+ sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarrayhive")
+
+ sql("create table structusingstructCarbon(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int> ) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table structusingstructCarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='name','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
+ sql("create table structusingstructhive(name struct<middlename:string, othernames:struct<firstname:string,lastname:string>,age:int> )row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by '&'")
+ sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table structusingstructhive")
+
}
+ test("select * from structusingarraycarbon") {
+ checkAnswer(sql("select * from structusingarraycarbon"),
+ sql("select * from structusingarrayhive"))
+ }
+
+ test("select * from structusingstructCarbon") {
+ checkAnswer(sql("select * from structusingstructCarbon"),
+ sql("select * from structusingstructhive"))
+ }
+
test("select * from complexcarbontable") {
checkAnswer(sql("select * from complexcarbontable"),
sql("select * from complexhivetable"))
@@ -112,8 +137,12 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("drop table if exists complexcarbontable")
- sql("drop table if exists complexhivetable")
+ sql("drop table if exists complexcarbontable").show
+ sql("drop table if exists complexhivetable").show
+ sql("drop table if exists structusingstructCarbon").show
+ sql("drop table if exists structusingstructHive").show
+ sql("drop table if exists structusingarraycarbon").show
+ sql("drop table if exists structusingarrayhive").show
}
}
[03/47] incubator-carbondata git commit: Problem: Value displayed as
Null after increase in precision for decimal datatype after aggregation
(#845)
Posted by ra...@apache.org.
Problem: Value displayed as Null after increase in precision for decimal datatype after aggregation (#845)
While creating a table if user specifies a precision for decimal datatype column and then executes an aggregation query on that column then in case if after aggregation the precision of the resulted value becomes greater than the user configure precision value, spark displays the aggregated value as null
Solution: Set the precision of the aggregated result as the precision for the decimal type while sending the aggregated result to spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6b7b41af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6b7b41af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6b7b41af
Branch: refs/heads/master
Commit: 6b7b41af98c161f28814aa4b08e5d2eb913217aa
Parents: a5714ac
Author: manishgupta88 <to...@gmail.com>
Authored: Thu Jul 21 21:55:40 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Thu Jul 21 21:55:40 2016 +0530
----------------------------------------------------------------------
.../carbondata/spark/agg/CarbonAggregates.scala | 21 +++++++++++++++++---
.../carbondata/spark/util/CarbonScalaUtil.scala | 12 +++++++++++
.../resources/decimalBoundaryDataCarbon.csv | 12 +++++++++++
.../test/resources/decimalBoundaryDataHive.csv | 11 ++++++++++
.../testsuite/bigdecimal/TestBigDecimal.scala | 18 +++++++++++++++++
5 files changed, 71 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6b7b41af/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
index 1a52688..de685ab 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
@@ -17,6 +17,8 @@
package org.carbondata.spark.agg
+import java.math.BigDecimal
+
import scala.language.implicitConversions
import org.apache.spark.sql.catalyst.InternalRow
@@ -26,6 +28,7 @@ import org.apache.spark.sql.types._
import org.carbondata.query.aggregator.MeasureAggregator
import org.carbondata.query.aggregator.impl._
+import org.carbondata.spark.util.CarbonScalaUtil
case class CountCarbon(child: Expression) extends UnaryExpression with PartialAggregate1 {
override def references: AttributeSet = child.references
@@ -366,7 +369,10 @@ case class AverageFunctionCarbon(expr: Expression, base: AggregateExpression1, f
} else {
avg match {
case avg: AvgBigDecimalAggregator =>
- Cast(Literal(avg.getBigDecimalValue), base.dataType).eval(null)
+ val decimalValue: BigDecimal = avg.getBigDecimalValue
+ val updatedDataType = CarbonScalaUtil
+ .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
+ Cast(Literal(decimalValue), updatedDataType).eval(null)
case avg: AvgLongAggregator =>
Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
case avg: AvgTimestampAggregator =>
@@ -483,7 +489,10 @@ case class SumFunctionCarbon(expr: Expression, base: AggregateExpression1, final
} else {
sum match {
case s: SumBigDecimalAggregator =>
- Cast(Literal(sum.getBigDecimalValue), base.dataType).eval(input)
+ val decimalValue: BigDecimal = sum.getBigDecimalValue
+ val updatedDataType = CarbonScalaUtil
+ .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
+ Cast(Literal(decimalValue), updatedDataType).eval(input)
case s: SumLongAggregator =>
Cast(Literal(sum.getLongValue), base.dataType).eval(input)
case _ =>
@@ -680,7 +689,13 @@ case class SumDisctinctFunctionCarbon(expr: Expression, base: AggregateExpressio
null
}
else {
- Cast(Literal(distinct.getValueObject), base.dataType).eval(null)
+ val updatedDataType = base.dataType match {
+ case decimal: DecimalType =>
+ CarbonScalaUtil
+ .getDecimalDataTypeWithUpdatedPrecision(distinct.getBigDecimalValue, base.dataType)
+ case _ => base.dataType
+ }
+ Cast(Literal(distinct.getValueObject), updatedDataType).eval(null)
}
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6b7b41af/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 6cd9986..20d45dc 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -105,6 +105,18 @@ object CarbonScalaUtil {
}
}
+ def getDecimalDataTypeWithUpdatedPrecision(decimalValue: java.math.BigDecimal,
+ currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = {
+ var newDataType: org.apache.spark.sql.types.DataType = currentDataType
+ if (null != decimalValue) {
+ val precision = decimalValue.precision
+ if (precision <= DecimalType.MAX_PRECISION) {
+ newDataType = DecimalType(precision, decimalValue.scale())
+ }
+ }
+ newDataType
+ }
+
case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6b7b41af/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv b/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
new file mode 100644
index 0000000..c64a9bf
--- /dev/null
+++ b/integration/spark/src/test/resources/decimalBoundaryDataCarbon.csv
@@ -0,0 +1,12 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.0000000000
+2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.0000000000
+3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.0000000000
+4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.0000000000
+5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.0000000000
+6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.0000000000
+7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.0000000000
+8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.0000000000
+9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.0000000000
+10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.0000000000
+11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.0000000000
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6b7b41af/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/decimalBoundaryDataHive.csv b/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
new file mode 100644
index 0000000..a2faaf1
--- /dev/null
+++ b/integration/spark/src/test/resources/decimalBoundaryDataHive.csv
@@ -0,0 +1,11 @@
+1,2015/7/23,china,aaa1,phone197,ASD69643,12345678901234510.0000000000
+2,2015/7/24,china,aaa2,phone756,ASD42892,12345678901234520.0000000000
+3,2015/7/25,china,aaa3,phone1904,ASD37014,12345678901234530.0000000000
+4,2015/7/26,china,aaa4,phone2435,ASD66902,12345678901234560.0000000000
+5,2015/7/27,china,aaa5,phone2441,ASD90633,22345678901234560.0000000000
+6,2015/7/28,china,aaa6,phone294,ASD59961,32345678901234560.0000000000
+7,2015/7/29,china,aaa7,phone610,ASD14875,42345678901234560.0000000000
+8,2015/7/30,china,aaa8,phone1848,ASD57308,52345678901234560.0000000000
+9,2015/7/18,china,aaa9,phone706,ASD86717,62345678901234560.0000000000
+10,2015/7/19,usa,aaa10,phone685,ASD30505,72345678901234560.0000000000
+11,2015/7/18,china,aaa11,phone1554,ASD26101,82345678901234560.0000000000
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6b7b41af/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index 3763e35..95dd1b4 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -108,6 +108,24 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select salary from carbonTable where salary<=45234525465882.24"),
sql("select salary from hiveTable where salary<=45234525465882.24"))
}
+
+ test("test aggregation on big decimal column with increased precision") {
+ sql("drop table if exists carbonBigDecimal")
+ sql("drop table if exists hiveBigDecimal")
+ sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10)) STORED BY 'org.apache.carbondata.format'")
+ sql("create table if not exists hiveBigDecimal(ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(27, 10))row format delimited fields terminated by ','")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
+ sql("LOAD DATA local inpath './src/test/resources/decimalBoundaryDataHive.csv' INTO table hiveBigDecimal")
+
+ checkAnswer(sql("select sum(salary) from carbonBigDecimal"),
+ sql("select sum(salary) from hiveBigDecimal"))
+
+ checkAnswer(sql("select sum(distinct salary) from carbonBigDecimal"),
+ sql("select sum(distinct salary) from hiveBigDecimal"))
+
+ sql("drop table if exists carbonBigDecimal")
+ sql("drop table if exists hiveBigDecimal")
+ }
override def afterAll {
[29/47] incubator-carbondata git commit: [CARBONDATA-113][Bug]fix
precision of DecimalType for CarbonAggregates(#875)
Posted by ra...@apache.org.
[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)
[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4d2f6842
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4d2f6842
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4d2f6842
Branch: refs/heads/master
Commit: 4d2f68427c6b3b2fc95175ebab16952e0999b939
Parents: adba597
Author: Gin-zhj <zh...@huawei.com>
Authored: Thu Jul 28 19:51:28 2016 +0800
Committer: david <qi...@qq.com>
Committed: Thu Jul 28 19:51:28 2016 +0800
----------------------------------------------------------------------
.../carbondata/spark/agg/CarbonAggregates.scala | 28 +++++++-------------
.../carbondata/spark/util/CarbonScalaUtil.scala | 16 +++++------
.../testsuite/bigdecimal/TestBigDecimal.scala | 23 +++++++++++++++-
3 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
index de685ab..50872ff 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
@@ -116,7 +116,7 @@ case class AverageCarbon(child: Expression, castedDataType: DataType = null)
AverageCarbonFinal(partialSum.toAttribute,
child.dataType match {
case IntegerType | StringType | LongType | TimestampType => DoubleType
- case _ => child.dataType
+ case _ => CarbonScalaUtil.updateDataType(child.dataType)
}),
partialSum :: Nil)
}
@@ -153,7 +153,11 @@ case class SumCarbon(child: Expression, castedDataType: DataType = null)
val partialSum = Alias(SumCarbon(child), "PartialSum")()
SplitEvaluation(
SumCarbonFinal(partialSum.toAttribute,
- if (castedDataType != null) castedDataType else child.dataType),
+ if (castedDataType != null) {
+ castedDataType
+ } else {
+ CarbonScalaUtil.updateDataType(child.dataType)
+ }),
partialSum :: Nil)
}
@@ -265,7 +269,7 @@ case class SumDistinctCarbon(child: Expression, castedDataType: DataType = null)
if (castedDataType != null) {
castedDataType
} else {
- child.dataType
+ CarbonScalaUtil.updateDataType(child.dataType)
}),
partialSum :: Nil)
}
@@ -369,10 +373,7 @@ case class AverageFunctionCarbon(expr: Expression, base: AggregateExpression1, f
} else {
avg match {
case avg: AvgBigDecimalAggregator =>
- val decimalValue: BigDecimal = avg.getBigDecimalValue
- val updatedDataType = CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
- Cast(Literal(decimalValue), updatedDataType).eval(null)
+ Cast(Literal(avg.getBigDecimalValue), base.dataType).eval(null)
case avg: AvgLongAggregator =>
Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
case avg: AvgTimestampAggregator =>
@@ -489,10 +490,7 @@ case class SumFunctionCarbon(expr: Expression, base: AggregateExpression1, final
} else {
sum match {
case s: SumBigDecimalAggregator =>
- val decimalValue: BigDecimal = sum.getBigDecimalValue
- val updatedDataType = CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
- Cast(Literal(decimalValue), updatedDataType).eval(input)
+ Cast(Literal(sum.getBigDecimalValue), base.dataType).eval(input)
case s: SumLongAggregator =>
Cast(Literal(sum.getLongValue), base.dataType).eval(input)
case _ =>
@@ -689,13 +687,7 @@ case class SumDisctinctFunctionCarbon(expr: Expression, base: AggregateExpressio
null
}
else {
- val updatedDataType = base.dataType match {
- case decimal: DecimalType =>
- CarbonScalaUtil
- .getDecimalDataTypeWithUpdatedPrecision(distinct.getBigDecimalValue, base.dataType)
- case _ => base.dataType
- }
- Cast(Literal(distinct.getValueObject), updatedDataType).eval(null)
+ Cast(Literal(distinct.getValueObject), base.dataType).eval(null)
}
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 20d45dc..b3effd3 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -105,19 +105,17 @@ object CarbonScalaUtil {
}
}
- def getDecimalDataTypeWithUpdatedPrecision(decimalValue: java.math.BigDecimal,
+ def updateDataType(
currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = {
- var newDataType: org.apache.spark.sql.types.DataType = currentDataType
- if (null != decimalValue) {
- val precision = decimalValue.precision
- if (precision <= DecimalType.MAX_PRECISION) {
- newDataType = DecimalType(precision, decimalValue.scale())
- }
+ currentDataType match {
+ case decimal: DecimalType =>
+ val scale = currentDataType.asInstanceOf[DecimalType].scale
+ DecimalType(DecimalType.MAX_PRECISION, scale)
+ case _ =>
+ currentDataType
}
- newDataType
}
-
case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
object CarbonSparkUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index b07705d..fb28568 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -136,7 +136,28 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists decimalDictLookUp")
}
-
+
+ test("test sum aggregation on big decimal column with high precision") {
+ sql("drop table if exists carbonBigDecimal")
+ sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
+
+ checkAnswer(sql("select sum(salary)+10 from carbonBigDecimal"),
+ sql("select sum(salary)+10 from hiveBigDecimal"))
+
+ sql("drop table if exists carbonBigDecimal")
+ }
+
+ test("test sum-distinct aggregation on big decimal column with high precision") {
+ sql("drop table if exists carbonBigDecimal")
+ sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into table carbonBigDecimal")
+
+ checkAnswer(sql("select sum(distinct(salary))+10 from carbonBigDecimal"),
+ sql("select sum(distinct(salary))+10 from hiveBigDecimal"))
+
+ sql("drop table if exists carbonBigDecimal")
+ }
override def afterAll {
sql("drop table if exists carbonTable")
[38/47] incubator-carbondata git commit: [Bug]Changed locking so that
zookeeper lock or hdfs lock will be available at executer. (#885)
Posted by ra...@apache.org.
[Bug]Changed locking so that zookeeper lock or hdfs lock will be available at executer. (#885)
[Bug]Changed locking so that zookeeper lock or hdfs lock will be available at executer. (#885)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f495b6bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f495b6bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f495b6bd
Branch: refs/heads/master
Commit: f495b6bd4653b7bb3139377cd93b4d16425553a7
Parents: 066f74b
Author: ashokblend <as...@gmail.com>
Authored: Fri Jul 29 09:19:03 2016 +0530
Committer: david <qi...@qq.com>
Committed: Fri Jul 29 11:49:03 2016 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 10 ++++++
.../org/carbondata/core/locks/HdfsFileLock.java | 9 +++++-
.../carbondata/core/locks/ZooKeeperLocking.java | 33 ++++++++++++++++++--
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 15 ++++++---
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 18 +++++++++--
.../spark/util/GlobalDictionaryUtil.scala | 11 +++++--
6 files changed, 82 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 86d87eb..b45d9b4 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -854,6 +854,16 @@ public final class CarbonCommonConstants {
*/
public static String COMPACTION_KEY_WORD = "COMPACTION";
+ /**
+ * hdfs temporary directory key
+ */
+ public static final String HDFS_TEMP_LOCATION = "hadoop.tmp.dir";
+
+ /**
+ * zookeeper url key
+ */
+ public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
index e8f44ed..702c8a1 100644
--- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
@@ -21,9 +21,12 @@ package org.carbondata.core.locks;
import java.io.DataOutputStream;
import java.io.IOException;
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.impl.FileFactory;
+import org.carbondata.core.util.CarbonProperties;
/**
* This class is used to handle the HDFS File locking.
@@ -31,6 +34,8 @@ import org.carbondata.core.datastorage.store.impl.FileFactory;
*/
public class HdfsFileLock extends AbstractCarbonLock {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HdfsFileLock.class.getName());
/**
* location hdfs file location
*/
@@ -41,7 +46,8 @@ public class HdfsFileLock extends AbstractCarbonLock {
public static String tmpPath;
static {
- tmpPath = System.getProperty("hadoop.tmp.dir");
+ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
+ System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
}
/**
@@ -51,6 +57,7 @@ public class HdfsFileLock extends AbstractCarbonLock {
public HdfsFileLock(String lockFileLocation, String lockFile) {
this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+ CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ LOGGER.info("HDFS lock path:"+this.location);
initRetry();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
index e1db3c6..3f309c3 100644
--- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
@@ -18,6 +18,7 @@
*/
package org.carbondata.core.locks;
+import java.io.File;
import java.util.Collections;
import java.util.List;
@@ -25,6 +26,7 @@ import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.util.CarbonProperties;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -69,7 +71,8 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
private String lockTypeFolder;
public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + '.' + tableIdentifier.getTableName(), lockFile);
+ this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
+ lockFile);
}
/**
@@ -80,7 +83,9 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
this.lockName = lockFile;
this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
- zk = ZookeeperInit.getInstance().getZookeeper();
+ String zooKeeperUrl =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
+ zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ CarbonCommonConstants.FILE_SEPARATOR + lockFile;
@@ -88,7 +93,7 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
createBaseNode();
// if exists returns null then path doesnt exist. so creating.
if (null == zk.exists(this.tableIdFolder, true)) {
- zk.create(this.tableIdFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ createRecursivly(this.tableIdFolder);
}
// if exists returns null then path doesnt exist. so creating.
if (null == zk.exists(this.lockTypeFolder, true)) {
@@ -111,6 +116,28 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
}
/**
+ * Create zookeepr node if not exist
+ * @param path
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ private void createRecursivly(String path) throws KeeperException, InterruptedException {
+ try {
+ if (zk.exists(path, true) == null && path.length() > 0) {
+ String temp = path.substring(0, path.lastIndexOf(File.separator));
+ createRecursivly(temp);
+ zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else {
+ return;
+ }
+ } catch (KeeperException e) {
+ throw e;
+ } catch (InterruptedException e) {
+ throw e;
+ }
+
+ }
+ /**
* Handling of the locking mechanism using zoo keeper.
*/
@Override public boolean lock() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index 5966240..7ec1b87 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -207,14 +207,19 @@ class CarbonMetastoreCatalog(hive: HiveContext, val storePath: String, client: C
// creating zookeeper instance once.
// if zookeeper is configured as carbon lock type.
- val zookeeperUrl: String = hive.getConf("spark.deploy.zookeeper.url", null)
+ val zookeeperUrl: String = hive.getConf(CarbonCommonConstants.ZOOKEEPER_URL, null)
if (zookeeperUrl != null) {
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperUrl)
ZookeeperInit.getInstance(zookeeperUrl)
LOGGER.info("Zookeeper url is configured. Taking the zookeeper as lock type.")
- CarbonProperties.getInstance
- .addProperty(CarbonCommonConstants.LOCK_TYPE,
- CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER
- )
+ var configuredLockType = CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.LOCK_TYPE)
+ if (null == configuredLockType) {
+ configuredLockType = CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.LOCK_TYPE,
+ configuredLockType)
+ }
}
if (metadataPath == null) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index e0e8cbf..a1870e9 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -38,6 +38,7 @@ import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
import org.carbondata.core.locks.CarbonLockFactory
import org.carbondata.core.locks.LockUsage
+import org.carbondata.core.util.CarbonProperties
import org.carbondata.core.util.CarbonTimeStatisticsFactory
import org.carbondata.spark.load.{CarbonLoaderUtil, CarbonLoadModel}
import org.carbondata.spark.partition.reader.{CSVParser, CSVReader}
@@ -154,7 +155,10 @@ case class DictionaryLoadModel(table: CarbonTableIdentifier,
highCardThreshold: Int,
rowCountPercentage: Double,
columnIdentifier: Array[ColumnIdentifier],
- isFirstLoad: Boolean) extends Serializable
+ isFirstLoad: Boolean,
+ hdfstemplocation: String,
+ lockType: String,
+ zooKeeperUrl: String) extends Serializable
case class ColumnDistinctValues(values: Array[String], rowCount: Long) extends Serializable
@@ -296,9 +300,16 @@ class CarbonGlobalDictionaryGenerateRDD(
val pathService = CarbonCommonFactory.getPathService
val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
model.hdfsLocation, model.table)
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
+ model.hdfstemplocation)
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE,
+ model.lockType)
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
+ model.zooKeeperUrl)
val dictLock = CarbonLockFactory
.getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
+ var isDictionaryLocked = false
// generate distinct value list
try {
val t1 = System.currentTimeMillis
@@ -328,7 +339,8 @@ class CarbonGlobalDictionaryGenerateRDD(
LOGGER.info("column " + model.table.getTableUniqueName + "." +
model.primDimensions(split.index).getColName + " is high cardinality column")
} else {
- if (dictLock.lockWithRetries()) {
+ isDictionaryLocked = dictLock.lockWithRetries()
+ if (isDictionaryLocked) {
logInfo(s"Successfully able to get the dictionary lock for ${
model.primDimensions(split.index).getColName
}")
@@ -397,7 +409,7 @@ class CarbonGlobalDictionaryGenerateRDD(
}
org.carbondata.core.util.CarbonUtil.clearDictionaryCache(dictionaryForSortIndexWriting);
if (dictLock != null) {
- if (dictLock.unlock()) {
+ if (isDictionaryLocked && dictLock.unlock()) {
logInfo(s"Dictionary ${
model.primDimensions(split.index).getColName
} Unlocked Successfully.")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f495b6bd/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index 18e777d..ff7e360 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -297,7 +297,11 @@ object GlobalDictionaryUtil extends Logging {
val dictFilePaths = dictDetail.dictFilePaths
val dictFileExists = dictDetail.dictFileExists
val columnIdentifier = dictDetail.columnIdentifiers
-
+ val hdfstemplocation = CarbonProperties.getInstance.
+ getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION)
+ val lockType = CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
+ val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
// load high cardinality identify configure
val highCardIdentifyEnable = CarbonProperties.getInstance().getProperty(
CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
@@ -326,7 +330,10 @@ object GlobalDictionaryUtil extends Logging {
highCardThreshold,
rowCountPercentage,
columnIdentifier,
- carbonLoadModel.getLoadMetadataDetails.size() == 0)
+ carbonLoadModel.getLoadMetadataDetails.size() == 0,
+ hdfstemplocation,
+ lockType,
+ zookeeperUrl)
}
/**
[08/47] incubator-carbondata git commit: [CARBONDATA-84] Change lock
framework to accept locklocation (#844)
Posted by ra...@apache.org.
[CARBONDATA-84] Change lock framework to accept locklocation (#844)
1.get dictionary lock location from CarbonTablePath
2.Change lock framework to accept lock location
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/584402e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/584402e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/584402e8
Branch: refs/heads/master
Commit: 584402e8008d64b57233dc5be2a13a2739e32c61
Parents: 07116d2
Author: ashokblend <as...@gmail.com>
Authored: Sat Jul 23 11:06:21 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 11:06:21 2016 +0530
----------------------------------------------------------------------
.../core/carbon/path/CarbonStorePath.java | 2 +-
.../core/carbon/path/CarbonTablePath.java | 18 +++++++++++++++-
.../core/locks/CarbonLockFactory.java | 22 ++++++++++++++++++++
.../org/carbondata/core/locks/HdfsFileLock.java | 16 ++++++++++----
.../carbondata/core/locks/LocalFileLock.java | 16 ++++++++++----
.../carbondata/core/locks/ZooKeeperLocking.java | 18 ++++++++--------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 9 ++++++--
7 files changed, 80 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
index c393f43..eac98e6 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
@@ -41,7 +41,7 @@ public class CarbonStorePath extends Path {
*/
public static CarbonTablePath getCarbonTablePath(String storePath,
CarbonTableIdentifier tableIdentifier) {
- CarbonTablePath carbonTablePath = new CarbonTablePath(
+ CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
+ tableIdentifier.getTableName());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
index 9ac8802..bd2c472 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
@@ -20,6 +20,7 @@ package org.carbondata.core.carbon.path;
import java.io.File;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
@@ -50,9 +51,16 @@ public class CarbonTablePath extends Path {
protected static final String INDEX_FILE_EXT = ".carbonindex";
protected String tablePath;
+ protected CarbonTableIdentifier carbonTableIdentifier;
- public CarbonTablePath(String tablePathString) {
+ /**
+ *
+ * @param carbonTableIdentifier
+ * @param tablePathString
+ */
+ public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
super(tablePathString);
+ this.carbonTableIdentifier = carbonTableIdentifier;
this.tablePath = tablePathString;
}
@@ -127,6 +135,14 @@ public class CarbonTablePath extends Path {
}
/**
+ * @return it return relative directory
+ */
+ public String getRelativeDictionaryDirectory() {
+ return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+ .getTableName();
+ }
+
+ /**
* This method will return the metadata directory location for a table
*
* @return
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
index dcf2c2c..8616c68 100644
--- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
@@ -62,6 +62,28 @@ public class CarbonLockFactory {
}
/**
+ *
+ * @param locFileLocation
+ * @param lockFile
+ * @return carbon lock
+ */
+ public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(locFileLocation, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
* This method will set the zookeeper status whether zookeeper to be used for locking or not.
*/
private static void updateZooKeeperLockingStatus() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
index e1fdd73..e8f44ed 100644
--- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
@@ -45,14 +45,22 @@ public class HdfsFileLock extends AbstractCarbonLock {
}
/**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public HdfsFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ initRetry();
+ }
+
+ /**
* @param tableIdentifier
* @param lockFile
*/
public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName()
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
initRetry();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
index 8194bfd..1120091 100644
--- a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
@@ -73,14 +73,22 @@ public class LocalFileLock extends AbstractCarbonLock {
}
/**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public LocalFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
+ this.lockFile = lockFile;
+ initRetry();
+ }
+
+ /**
* @param tableIdentifier
* @param lockFile
*/
public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this.location =
- tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName();
- this.lockFile = lockFile;
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
initRetry();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
index 47349ad..e1db3c6 100644
--- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
@@ -68,22 +68,22 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
private String lockTypeFolder;
+ public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + '.' + tableIdentifier.getTableName(), lockFile);
+ }
+
/**
- * @param tableIdentifier
+ * @param lockLocation
* @param lockFile
*/
- public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ public ZooKeeperLocking(String lockLocation, String lockFile) {
this.lockName = lockFile;
- this.tableIdFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName();
+ this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
zk = ZookeeperInit.getInstance().getZookeeper();
- this.lockTypeFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName() + CarbonCommonConstants.FILE_SEPARATOR
- + lockFile;
+ this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
try {
createBaseNode();
// if exists returns null then path doesnt exist. so creating.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 9abcceb..e519030 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -30,6 +30,7 @@ import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
+import org.carbondata.common.factory.CarbonCommonFactory
import org.carbondata.common.logging.LogServiceFactory
import org.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
import org.carbondata.core.carbon.metadata.datatype.DataType
@@ -295,8 +296,12 @@ class CarbonGlobalDictionaryGenerateRDD(
var dictionaryForDistinctValueLookUp: org.carbondata.core.cache.dictionary.Dictionary = _
var dictionaryForSortIndexWriting: org.carbondata.core.cache.dictionary.Dictionary = _
var dictionaryForDistinctValueLookUpCleared: Boolean = false
- val dictLock = CarbonLockFactory.getCarbonLockObj(model.table,
- model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
+ val pathService = CarbonCommonFactory.getPathService
+ val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
+ model.hdfsLocation, model.table)
+ val dictLock = CarbonLockFactory
+ .getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
+ model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
// generate distinct value list
try {
val t1 = System.currentTimeMillis
[47/47] incubator-carbondata git commit: Merge PRs from old github
This closes #58
Posted by ra...@apache.org.
Merge PRs from old github This closes #58
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/80cfa39c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/80cfa39c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/80cfa39c
Branch: refs/heads/master
Commit: 80cfa39c6d784883fd14c87774ade88bbea57ff8
Parents: 10ed89a 21d8c7e
Author: ravipesala <ra...@gmail.com>
Authored: Mon Aug 1 15:34:11 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Aug 1 15:34:11 2016 +0530
----------------------------------------------------------------------
.../core/carbon/ColumnIdentifier.java | 4 +
.../carbon/datastore/SegmentTaskIndexStore.java | 2 +-
.../metadata/schema/table/CarbonTable.java | 48 ++--
.../schema/table/column/CarbonDimension.java | 4 +
.../core/carbon/path/CarbonStorePath.java | 2 +-
.../core/carbon/path/CarbonTablePath.java | 18 +-
.../carbon/querystatistics/QueryStatistic.java | 12 +
.../core/constants/CarbonCommonConstants.java | 29 ++
.../org/carbondata/core/load/BlockDetails.java | 8 +-
.../core/load/LoadMetadataDetails.java | 20 ++
.../core/util/CarbonLoadStatisticsDummy.java | 12 +-
.../core/util/CarbonLoadStatisticsImpl.java | 65 ++---
.../org/carbondata/core/util/DataTypeUtil.java | 97 +++++--
.../carbondata/core/util/LoadStatistics.java | 6 +-
.../scan/executor/util/QueryUtil.java | 55 ++--
.../scan/filter/FilterExpressionProcessor.java | 20 +-
.../org/carbondata/scan/filter/FilterUtil.java | 32 +++
.../filter/resolver/AndFilterResolverImpl.java | 5 +-
.../resolver/LogicalFilterResolverImpl.java | 11 +-
.../visitor/CustomTypeDictionaryVisitor.java | 8 +
.../visitor/DictionaryColumnVisitor.java | 4 +-
.../examples/ComplexTypeExample.scala | 6 +-
.../hadoop/test/util/StoreCreator.java | 2 +-
.../spark/sql/common/util/CsvCompare.scala | 1 -
.../spark/merger/CompactionCallable.java | 36 +--
.../carbondata/spark/load/CarbonLoaderUtil.java | 22 +-
.../spark/merger/CarbonDataMergerUtil.java | 84 ++++--
.../spark/sql/CarbonDatasourceRelation.scala | 24 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 6 +-
.../org/apache/spark/sql/CarbonSqlParser.scala | 216 +++++++++------
.../spark/sql/SparkUnknownExpression.scala | 24 +-
.../execution/command/carbonTableSchema.scala | 34 ++-
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 27 +-
.../scala/org/apache/spark/util/FileUtils.scala | 6 +-
.../org/apache/spark/util/SplitUtils.scala | 6 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 65 +++--
.../spark/rdd/CarbonDataRDDFactory.scala | 268 ++++++++++---------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 35 ++-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 183 +++++++------
.../carbondata/spark/rdd/CarbonScanRDD.scala | 51 +++-
.../org/carbondata/spark/rdd/Compactor.scala | 79 +++---
.../spark/tasks/DictionaryWriterTask.scala | 56 ++--
.../carbondata/spark/util/CarbonScalaUtil.scala | 13 +
.../spark/util/GlobalDictionaryUtil.scala | 16 +-
integration/spark/src/test/resources/array1.csv | 2 +
.../resources/complexTypeDecimalNestedHive.csv | 8 +
.../spark/src/test/resources/datadelimiter.csv | 11 +
.../src/test/resources/datanullmeasurecol.csv | 3 +
.../spark/src/test/resources/datasample.csv | 1 +
.../resources/decimalBoundaryDataCarbon.csv | 12 +
.../test/resources/decimalBoundaryDataHive.csv | 11 +
.../test/resources/emptyDimensionDataHive.csv | 20 ++
.../spark/src/test/resources/struct_all.csv | 4 +
.../src/test/resources/structusingstruct.csv | 2 +
.../spark/src/test/resources/timestampdata.csv | 2 +
.../apache/spark/sql/TestCarbonSqlParser.scala | 72 ++---
...plexPrimitiveTimestampDirectDictionary.scala | 12 +
.../complexType/TestComplexTypeQuery.scala | 52 +++-
.../dataload/TestLoadDataWithJunkChars.scala | 43 +++
.../dataload/TestLoadDataWithNullMeasures.scala | 1 +
.../testsuite/bigdecimal/TestBigDecimal.scala | 52 +++-
.../createtable/TestCreateTableSyntax.scala | 29 +-
.../DataCompactionCardinalityBoundryTest.scala | 6 +-
.../datacompaction/DataCompactionTest.scala | 6 +-
.../MajorCompactionIgnoreInMinorTest.scala | 130 +++++++++
.../MajorCompactionStopsAfterCompaction.scala | 125 +++++++++
.../TestDataWithDicExcludeAndInclude.scala | 22 +-
.../dataload/TestLoadDataWithHiveSyntax.scala | 59 +++-
.../dataload/TestLoadDataWithNoMeasure.scala | 2 +-
.../TestLoadDataWithNotProperInputFile.scala | 6 +-
.../NoDictionaryColumnTestCase.scala | 6 +-
...estampDataTypeDirectDictionaryTestCase.scala | 61 ++++-
...TypeDirectDictionaryWithNoDictTestCase.scala | 6 +-
.../TimestampDataTypeNullDataTest.scala | 2 +-
.../filterexpr/AllDataTypesTestCaseFilter.scala | 14 +-
.../filterexpr/CountStarTestCase.scala | 72 +++++
.../NullMeasureValueTestCaseFilter.scala | 7 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 2 +-
.../util/GlobalDictionaryUtilTestCase.scala | 4 +-
.../carbondata/lcm/locks/CarbonLockFactory.java | 27 +-
.../org/carbondata/lcm/locks/HdfsFileLock.java | 25 +-
.../org/carbondata/lcm/locks/LocalFileLock.java | 16 +-
.../carbondata/lcm/locks/ZooKeeperLocking.java | 49 +++-
.../lcm/status/SegmentStatusManager.java | 2 -
.../processing/csvload/GraphExecutionUtil.java | 31 +++
.../csvreaderstep/CustomDataStream.java | 108 ++++++++
.../processing/csvreaderstep/CustomReader.java | 157 -----------
.../csvreaderstep/UnivocityCsvParser.java | 13 +-
.../processing/datatypes/PrimitiveDataType.java | 28 +-
.../processing/mdkeygen/MDKeyGenStep.java | 2 +-
.../processing/mdkeygen/MDKeyGenStepMeta.java | 3 +-
.../sortandgroupby/sortdata/SortDataRows.java | 2 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 21 ++
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 3 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 72 ++++-
.../FileStoreSurrogateKeyGenForCSV.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 7 +-
.../processing/util/CarbonSchemaParser.java | 1 +
98 files changed, 2222 insertions(+), 937 deletions(-)
----------------------------------------------------------------------
[35/47] incubator-carbondata git commit: [CARBONDATA-114][Bug] While
describing/querying the complex type dimension precision not considered
(#879)
Posted by ra...@apache.org.
[CARBONDATA-114][Bug] While describing/querying the complex type dimension precision not considered (#879)
For decimal columns system was not able to maintain the scale/precision, always default precision has been taken which needs to be handled.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/61b60746
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/61b60746
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/61b60746
Branch: refs/heads/master
Commit: 61b6074605445d5b9d92b31be85c3f4f9fc8a739
Parents: b3779e4
Author: sujith71955 <su...@gmail.com>
Authored: Fri Jul 29 06:01:27 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jul 29 06:01:27 2016 +0530
----------------------------------------------------------------------
.../spark/sql/CarbonDatasourceRelation.scala | 22 +++++++++++++-------
.../createtable/TestCreateTableSyntax.scala | 16 +++++++++++++-
2 files changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/61b60746/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index ed4e6cb..885ad49 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -172,7 +172,7 @@ case class CarbonRelation(
childDim.getDataType.toString.toLowerCase match {
case "array" => s"array<${ getArrayChildren(childDim.getColName) }>"
case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>"
- case dType => dType
+ case dType => addDecimalScaleAndPrecision(childDim, dType)
}
}).mkString(",")
}
@@ -188,7 +188,8 @@ case class CarbonRelation(
}:struct<${ metaData.carbonTable.getChildren(childDim.getColName)
.asScala.map(f => s"${ recursiveMethod(childDim.getColName, f) }").mkString(",")
}>"
- case dType => s"${ childDim.getColName.substring(dimName.length() + 1) }:${ dType }"
+ case dType => s"${ childDim.getColName
+ .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
}
}).mkString(",")
}
@@ -212,12 +213,7 @@ case class CarbonRelation(
case "struct" => CarbonMetastoreTypes
.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
case dType =>
- var dataType = dType
- if (dimval.getDataType == org.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) {
- dataType +=
- "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema
- .getScale + ")"
- }
+ var dataType = addDecimalScaleAndPrecision(dimval, dType)
CarbonMetastoreTypes.toDataType(dataType)
}
@@ -257,6 +253,16 @@ case class CarbonRelation(
}
}
+ def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
+ var dType = dataType
+ if (dimval.getDataType == org.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) {
+ dType +=
+ "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema
+ .getScale + ")"
+ }
+ dType
+ }
+
private var tableStatusLastUpdateTime = 0L
private var sizeInBytesLocalValue = 0L
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/61b60746/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
index 3868928..f62a3a1 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
@@ -21,7 +21,7 @@ package org.carbondata.spark.testsuite.createtable
import org.apache.spark.sql.common.util.CarbonHiveContext._
import org.apache.spark.sql.common.util.QueryTest
-
+import org.apache.spark.sql.{CarbonContext, Row}
import org.carbondata.spark.exception.MalformedCarbonCommandException
import org.scalatest.BeforeAndAfterAll
@@ -99,6 +99,20 @@ class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists hivetable")
}
+ test("describe command carbon table for decimal scale and precision test") {
+ sql("create table carbontablePrecision(id int, name string, dept string, mobile array<string>, "+
+ "country string, salary decimal(10,6)) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='salary,id')")
+ checkAnswer(
+ sql("describe carbontablePrecision"),
+ Seq(Row("country","string",""),
+ Row("dept","string",""),Row("id","int",""),Row("mobile","array<string>",""),Row("name","string",""),
+ Row("salary","decimal(10,6)","")
+ )
+ )
+ sql("drop table if exists carbontablePrecision")
+ }
+
test("create carbon table without dimensions") {
try {
sql("create table carbontable(msr1 int, msr2 double, msr3 bigint, msr4 decimal)" +
[18/47] incubator-carbondata git commit: [CARBONDATA-105][Bug] In
case of concurrent data loading,
pre calculation of existence of dictionary file will not have proper result.
(#866)
Posted by ra...@apache.org.
[CARBONDATA-105][Bug] In case of concurrent data loading, pre calculation of existence of dictionary file will not have proper result. (#866)
Check added after acquiring lock
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8289525d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8289525d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8289525d
Branch: refs/heads/master
Commit: 8289525d286d2d4e135c4405097a45d105b48a1a
Parents: f57a95c
Author: ashokblend <as...@gmail.com>
Authored: Mon Jul 25 14:42:10 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 14:42:10 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 +++
.../spark/util/GlobalDictionaryUtilConcurrentTestCase.scala | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8289525d/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 7067fd3..9a94c5b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -342,6 +342,9 @@ class CarbonGlobalDictionaryGenerateRDD(
} is locked for updation. Please try after some time")
}
val t2 = System.currentTimeMillis
+ val fileType = FileFactory.getFileType(model.dictFilePaths(split.index))
+ model.dictFileExists(split.index) = FileFactory
+ .isFileExist(model.dictFilePaths(split.index), fileType)
dictionaryForDistinctValueLookUp = if (model.dictFileExists(split.index)) {
CarbonLoaderUtil.getDictionary(model.table,
model.columnIdentifier(split.index),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8289525d/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 60e9281..9230c6d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -143,7 +143,7 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
val sortIndexFile = FileFactory.getCarbonFile(sortIndexPath, FileFactory.getFileType(sortIndexPath))
assert(sortIndexFile.exists())
val sortIndexFiles = carbonTablePath.getSortIndexFiles(sortIndexFile.getParentFile, columnIdentifier.getColumnId)
- assert(sortIndexFiles.length == 2)
+ assert(sortIndexFiles.length >= 1)
deleteFiles(files)
}
[19/47] incubator-carbondata git commit: [CARBONDATA-103][Bug] Rename
CreateCube to CreateTable to correct the audit log of create table command
(#865)
Posted by ra...@apache.org.
[CARBONDATA-103][Bug] Rename CreateCube to CreateTable to correct the audit log of create table command (#865)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bb8b0fdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bb8b0fdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bb8b0fdc
Branch: refs/heads/master
Commit: bb8b0fdce1db0d0a2d00e7c9b9969635ed3c1b50
Parents: 8289525
Author: Mohammad Shahid Khan <mo...@gmail.com>
Authored: Mon Jul 25 14:47:42 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 14:47:42 2016 +0530
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala | 2 +-
.../org/apache/spark/sql/execution/command/carbonTableSchema.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bb8b0fdc/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 4fd5f14..79471a9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -435,7 +435,7 @@ class CarbonSqlParser()
tableProperties)
// get logical plan.
- CreateCube(tableModel)
+ CreateTable(tableModel)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bb8b0fdc/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 0278a68..30c022e 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -1254,7 +1254,7 @@ private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) e
}
}
-private[sql] case class CreateCube(cm: tableModel) extends RunnableCommand {
+private[sql] case class CreateTable(cm: tableModel) extends RunnableCommand {
def run(sqlContext: SQLContext): Seq[Row] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
[46/47] incubator-carbondata git commit: Fixed issues after merge
Posted by ra...@apache.org.
Fixed issues after merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/21d8c7ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/21d8c7ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/21d8c7ea
Branch: refs/heads/master
Commit: 21d8c7ea6884d820605a187dcf2bcb6787aa09a6
Parents: 50dfdf6
Author: ravipesala <ra...@gmail.com>
Authored: Mon Aug 1 15:12:44 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Aug 1 15:12:44 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/core/util/DataTypeUtil.java | 44 ++++++++---------
.../aggregator/impl/CountStarAggregator.java | 51 --------------------
.../scan/executor/util/QueryUtil.java | 1 -
.../scan/filter/FilterExpressionProcessor.java | 1 -
.../filter/resolver/AndFilterResolverImpl.java | 1 +
.../resolver/LogicalFilterResolverImpl.java | 1 +
.../spark/sql/CarbonDictionaryDecoder.scala | 6 ++-
.../execution/command/carbonTableSchema.scala | 6 ++-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 2 -
...estampDataTypeDirectDictionaryTestCase.scala | 20 +++++---
.../filterexpr/AllDataTypesTestCaseFilter.scala | 10 ++--
12 files changed, 53 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
index 995bf17..92316d5 100644
--- a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
@@ -29,6 +29,7 @@ import java.util.Date;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.carbondata.core.constants.CarbonCommonConstants;
@@ -268,7 +269,7 @@ public final class DataTypeUtil {
scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal);
org.apache.spark.sql.types.Decimal decConverter =
new org.apache.spark.sql.types.Decimal();
- return decConverter.set(scalaDecVal, 19, 2);
+ return decConverter.set(scalaDecVal);
default:
return UTF8String.fromString(data);
}
@@ -307,39 +308,36 @@ public final class DataTypeUtil {
}
/**
- * This method will parse a given string value corresponding to its datatype
+ * Below method will be used to basically to know whether any non parseable
+ * data is present or not. if present then return null so that system can
+ * process to default null member value.
*
- * @param value value to parse
- * @param dataType datatype for that value
- * @return
+ * @param data data
+ * @param actualDataType actual data type
+ * @return actual data after conversion
*/
- public static boolean validateColumnValueForItsDataType(String value, DataType dataType) {
+ public static Object normalizeIntAndLongValues(String data, DataType actualDataType) {
+ if (null == data) {
+ return null;
+ }
try {
Object parsedValue = null;
- // validation will not be done for timestamp datatype as for timestamp direct dictionary
- // is generated. No dictionary file is created for timestamp datatype column
- switch (dataType) {
- case DECIMAL:
- parsedValue = new BigDecimal(value);
- break;
+ switch (actualDataType) {
case INT:
- parsedValue = Integer.parseInt(value);
+ parsedValue = Integer.parseInt(data);
break;
case LONG:
- parsedValue = Long.valueOf(value);
- break;
- case DOUBLE:
- parsedValue = Double.valueOf(value);
+ parsedValue = Long.parseLong(data);
break;
default:
- return true;
+ return data;
}
- if (null != parsedValue) {
- return true;
+ if(null != parsedValue) {
+ return data;
}
- return false;
- } catch (Exception e) {
- return false;
+ return null;
+ } catch (NumberFormatException ex) {
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java b/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
deleted file mode 100644
index 3b7e78c..0000000
--- a/core/src/main/java/org/carbondata/query/aggregator/impl/CountStarAggregator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.query.aggregator.impl;
-
-import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
-import org.carbondata.query.aggregator.MeasureAggregator;
-
-/**
- * Class Description : It will return total count of values
- */
-public class CountStarAggregator extends CountAggregator {
-
- @Override public void agg(double newVal) {
- super.agg(newVal);
- }
-
- @Override public void agg(Object newVal) {
- aggVal++;
- }
-
- @Override public void agg(MeasureColumnDataChunk dataChunk, int index) {
- aggVal++;
- }
-
- @Override public MeasureAggregator getCopy() {
- CountStarAggregator aggregator = new CountStarAggregator();
- aggregator.aggVal = aggVal;
- return aggregator;
- }
-
- @Override public MeasureAggregator getNew() {
- return new CountStarAggregator();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
index 8fee45a..00e0f48 100644
--- a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
@@ -57,7 +57,6 @@ import org.carbondata.scan.executor.exception.QueryExecutionException;
import org.carbondata.scan.executor.infos.KeyStructureInfo;
import org.carbondata.scan.expression.ColumnExpression;
import org.carbondata.scan.expression.Expression;
-import org.carbondata.scan.expression.logical.BinaryLogicalExpression;
import org.carbondata.scan.filter.GenericQueryType;
import org.carbondata.scan.filter.resolver.FilterResolverIntf;
import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
index a2c6f28..6543af6 100644
--- a/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
@@ -40,7 +40,6 @@ import org.carbondata.scan.expression.Expression;
import org.carbondata.scan.expression.conditional.BinaryConditionalExpression;
import org.carbondata.scan.expression.conditional.ConditionalExpression;
import org.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.carbondata.scan.expression.logical.BinaryLogicalExpression;
import org.carbondata.scan.filter.executer.FilterExecuter;
import org.carbondata.scan.filter.intf.ExpressionType;
import org.carbondata.scan.filter.resolver.ConditionalFilterResolverImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java b/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
index 4f7d814..1be7595 100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
@@ -23,6 +23,7 @@ import java.util.SortedMap;
import org.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.BinaryExpression;
import org.carbondata.scan.filter.intf.ExpressionType;
public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
index 4aa448f..8b18a21 100644
--- a/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -24,6 +24,7 @@ import java.util.SortedMap;
import org.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.BinaryExpression;
import org.carbondata.scan.expression.Expression;
import org.carbondata.scan.filter.intf.ExpressionType;
import org.carbondata.scan.filter.intf.FilterExecuterType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 9e66d99..d0d4f16 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -101,7 +101,11 @@ case class CarbonDictionaryDecoder(
case DataType.DECIMAL =>
val scale: Int = carbonDimension.getColumnSchema.getScale
val precision: Int = carbonDimension.getColumnSchema.getPrecision
- DecimalType(18, 2)
+ if (scale == 0 && precision == 0) {
+ DecimalType(18, 2)
+ } else {
+ DecimalType(precision, scale)
+ }
case DataType.TIMESTAMP => TimestampType
case DataType.STRUCT =>
CarbonMetastoreTypes
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 01bb218..d19d1c5 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -49,10 +49,12 @@ import org.carbondata.core.carbon.metadata.schema.table.{CarbonTable, TableInfo,
import org.carbondata.core.carbon.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
+import org.carbondata.core.load.LoadMetadataDetails
import org.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.carbondata.integration.spark.merger.CompactionType
import org.carbondata.lcm.locks.{CarbonLockFactory, LockUsage}
import org.carbondata.lcm.status.SegmentStatusManager
+import org.carbondata.processing.etl.DataLoadingException
import org.carbondata.spark.CarbonSparkFactory
import org.carbondata.spark.exception.MalformedCarbonCommandException
import org.carbondata.spark.load._
@@ -1193,10 +1195,10 @@ private[sql] case class LoadTable(
}
} catch {
case dle: DataLoadingException =>
- LOGGER.audit(s"Dataload failed for $schemaName.$tableName. " + dle.getMessage)
+ LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + dle.getMessage)
throw dle
case mce: MalformedCarbonCommandException =>
- LOGGER.audit(s"Dataload failed for $schemaName.$tableName. " + mce.getMessage)
+ LOGGER.audit(s"Dataload failed for $dbName.$tableName. " + mce.getMessage)
throw mce
} finally {
if (carbonLock != null) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 748a408..70e8257 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -19,7 +19,7 @@
package org.carbondata.spark.rdd
import java.util
-import java.util.concurrent.{Executors, ExecutorService, Future}
+import java.util.concurrent._
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
@@ -31,7 +31,7 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileSplit
import org.apache.spark.{Logging, Partition, SparkContext, SparkEnv}
import org.apache.spark.sql.{CarbonEnv, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, Partitioner}
+import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionCallableModel, CompactionModel, Partitioner}
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.util.{FileUtils, SplitUtils}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index e79937f..9e0de52 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -36,8 +36,6 @@ import org.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
-import org.carbondata.core.locks.CarbonLockFactory
-import org.carbondata.core.locks.LockUsage
import org.carbondata.core.util.CarbonProperties
import org.carbondata.core.util.CarbonTimeStatisticsFactory
import org.carbondata.lcm.locks.{CarbonLockFactory, LockUsage}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index a29cefb..dc7d6a7 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -51,11 +51,17 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
)
CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true")
sql(
- "CREATE TABLE directDictionaryTable (empno int,doj Timestamp, " +
+ "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, " +
"salary int) " +
"STORED BY 'org.apache.carbondata.format'"
)
+ sql(
+ "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp, " +
+ "salary int) " +
+ "row format delimited fields terminated by ','"
+ )
+
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
@@ -63,6 +69,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive");
} catch {
case x: Throwable => CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
@@ -80,7 +87,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
test("test direct dictionary for getting all the values") {
checkAnswer(
- sql("select doj from directDictionaryCube"),
+ sql("select doj from directDictionaryTable"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
Row(null)
@@ -90,7 +97,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
test("test direct dictionary for not equals condition") {
checkAnswer(
- sql("select doj from directDictionaryCube where doj != '2016-04-14 15:00:09.0'"),
+ sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09.0'"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
)
)
@@ -98,7 +105,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
test("test direct dictionary for null condition") {
checkAnswer(
- sql("select doj from directDictionaryCube where doj is null"),
+ sql("select doj from directDictionaryTable where doj is null"),
Seq(Row(null)
)
)
@@ -111,7 +118,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
)
}
-
+
test("select doj from directDictionaryTable with regexp_replace equals filter") {
checkAnswer(
sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') = '2016/03/14 15:00:09'"),
@@ -122,7 +129,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
test("select doj from directDictionaryTable with regexp_replace NOT IN filter") {
checkAnswer(
sql("select doj from directDictionaryTable where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')"),
- Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")), Row(null))
+ sql("select doj from directDictionaryTable_hive where regexp_replace(doj, '-', '/') NOT IN ('2016/03/14 15:00:09')")
)
}
@@ -142,6 +149,7 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
override def afterAll {
sql("drop table directDictionaryTable")
+ sql("drop table directDictionaryTable_hive")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/21d8c7ea/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
index 4075e60..3e86e16 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/AllDataTypesTestCaseFilter.scala
@@ -47,16 +47,16 @@ class AllDataTypesTestCaseFilter extends QueryTest with BeforeAndAfterAll {
sql("select empno,empname,utilization,count(salary),sum(empno) from alldatatypestableFilter_hive where empname in ('arvind','ayushi') group by empno,empname,utilization"))
}
- test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')") {
+ test("select empno,empname from alldatatypestableFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')") {
checkAnswer(
- sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"),
- sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"))
+ sql("select empno,empname from alldatatypestableFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"),
+ sql("select empno,empname from alldatatypestableFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') NOT IN ('development')"))
}
test("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'") {
checkAnswer(
- sql("select empno,empname from alldatatypescubeFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"),
- sql("select empno,empname from alldatatypescubeFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"))
+ sql("select empno,empname from alldatatypestableFilter where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"),
+ sql("select empno,empname from alldatatypestableFilter_hive where regexp_replace(workgroupcategoryname, 'er', 'ment') != 'development'"))
}
override def afterAll {
[41/47] incubator-carbondata git commit: Problem: Exception thrown
while executing drop table command in spark-sql cli (#888)
Posted by ra...@apache.org.
Problem: Exception thrown while executing drop table command in spark-sql cli (#888)
Analysis: Whenever drop table operation is performed, first the cube is deleted from carbon metastore and then command is executed to delete the same table from hive metastore. While deleting from hive metastore the call comes back to carbon metastore catalog where we again check for table existence but table is not found and exception is thrown. Call comes again to carbon because the command is executed as spark sql command and not as hive sql command and we catch the logical plan when drop command is executed as spark sql command.
Impact: drop table operation
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b82a960b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b82a960b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b82a960b
Branch: refs/heads/master
Commit: b82a960bd58acd83d1f6ecf79340128170178414
Parents: 87f0dec
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Jul 29 15:31:33 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Fri Jul 29 18:01:33 2016 +0800
----------------------------------------------------------------------
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 12 +---------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 14 ++++++------
.../spark/util/GlobalDictionaryUtil.scala | 6 ++---
.../resources/complexTypeDecimalNestedHive.csv | 8 +++++++
.../test/resources/emptyDimensionDataHive.csv | 20 +++++++++++++++++
.../dataload/TestLoadDataWithNullMeasures.scala | 1 +
.../TestDataWithDicExcludeAndInclude.scala | 22 ++++++++++++++-----
.../dataload/TestLoadDataWithHiveSyntax.scala | 23 +++++++++++++++-----
.../NoDictionaryColumnTestCase.scala | 6 +++--
.../NullMeasureValueTestCaseFilter.scala | 7 +++---
10 files changed, 82 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index 7ec1b87..e18e1d3 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -495,17 +495,7 @@ class CarbonMetastoreCatalog(hive: HiveContext, val storePath: String, client: C
org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
.removeTable(schemaName + "_" + cubeName)
- try {
- sqlContext.sql(s"DROP TABLE $schemaName.$cubeName").collect()
- } catch {
- case e: Exception =>
- LOGGER.audit(
- s"Error While deleting the table $schemaName.$cubeName during drop Table" + e.getMessage)
- }
-
- logInfo(s"Table $cubeName of $schemaName Database dropped syccessfully.")
- LOGGER.info("Table " + cubeName + " of " + schemaName + " Database dropped syccessfully.")
-
+ sqlContext.asInstanceOf[HiveContext].runSqlHive(s"DROP TABLE IF EXISTS $schemaName.$cubeName")
}
private def getTimestampFileAndType(schemaName: String, cubeName: String) = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index a1870e9..70d0cb1 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -156,7 +156,7 @@ case class DictionaryLoadModel(table: CarbonTableIdentifier,
rowCountPercentage: Double,
columnIdentifier: Array[ColumnIdentifier],
isFirstLoad: Boolean,
- hdfstemplocation: String,
+ hdfsTempLocation: String,
lockType: String,
zooKeeperUrl: String) extends Serializable
@@ -301,11 +301,11 @@ class CarbonGlobalDictionaryGenerateRDD(
val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
model.hdfsLocation, model.table)
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION,
- model.hdfstemplocation)
+ model.hdfsTempLocation)
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE,
- model.lockType)
- CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
- model.zooKeeperUrl)
+ model.lockType)
+ CarbonProperties.getInstance.addProperty(CarbonCommonConstants.ZOOKEEPER_URL,
+ model.zooKeeperUrl)
val dictLock = CarbonLockFactory
.getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
@@ -408,8 +408,8 @@ class CarbonGlobalDictionaryGenerateRDD(
.clearDictionaryCache(dictionaryForDistinctValueLookUp);
}
org.carbondata.core.util.CarbonUtil.clearDictionaryCache(dictionaryForSortIndexWriting);
- if (dictLock != null) {
- if (isDictionaryLocked && dictLock.unlock()) {
+ if (dictLock != null && isDictionaryLocked) {
+ if (dictLock.unlock()) {
logInfo(s"Dictionary ${
model.primDimensions(split.index).getColName
} Unlocked Successfully.")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index 0e15803..1c45230 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -297,8 +297,8 @@ object GlobalDictionaryUtil extends Logging {
val dictFilePaths = dictDetail.dictFilePaths
val dictFileExists = dictDetail.dictFileExists
val columnIdentifier = dictDetail.columnIdentifiers
- val hdfstemplocation = CarbonProperties.getInstance.getProperty(
- CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
+ val hdfsTempLocation = CarbonProperties.getInstance.
+ getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
val lockType = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
@@ -331,7 +331,7 @@ object GlobalDictionaryUtil extends Logging {
rowCountPercentage,
columnIdentifier,
carbonLoadModel.getLoadMetadataDetails.size() == 0,
- hdfstemplocation,
+ hdfsTempLocation,
lockType,
zookeeperUrl)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
new file mode 100644
index 0000000..ae67723
--- /dev/null
+++ b/integration/spark/src/test/resources/complexTypeDecimalNestedHive.csv
@@ -0,0 +1,8 @@
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113:imei$3.33:imsi
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123:imei$7.33:imsi
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123:imei$56.33:imsi
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133:imei$5.33:imsi
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133:imei$54.33:imsi
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13:imei$56.33:imsi
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213:imei$44.33:imsi
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13:imei$33.33:imsi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/emptyDimensionDataHive.csv b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
new file mode 100644
index 0000000..2e89887
--- /dev/null
+++ b/integration/spark/src/test/resources/emptyDimensionDataHive.csv
@@ -0,0 +1,20 @@
+1,2015/7/23,,aaa1,,,15000
+2,2015/7/24,,,,,15001
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
+4,2015/7/26,china,aaa4,,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,,15004
+6,2015/7/28,,aaa6,phone294,ASD59961,15005
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006
+8,2015/7/30,china,aaa8,,ASD57308,15007
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
+11,2015/7/18,china,aaa11,phone1554,ASD26101,15010
+12,2015/7/19,,aaa12,phone1781,ASD85711,15011
+13,2015/7/20,china,,phone943,,15012
+14,2015/7/21,china,aaa14,phone1954,ASD80468,15013
+15,2015/7/22,china,aaa15,phone451,ASD1954,15014
+16,2015/7/23,china,aaa16,phone390,ASD38513,15015
+17,2015/7/24,,aaa17,phone1929,ASD86213,15016
+18,2015/7/25,usa,aaa18,,ASD88812,15017
+19,2015/7/26,china,aaa19,phone2151,ASD9316,15018
+20,2015/7/27,china,aaa20,phone2625,ASD62597,15019
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
index 09f44ad..3a920b8 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithNullMeasures.scala
@@ -29,6 +29,7 @@ import org.scalatest.BeforeAndAfterAll
*/
class TestLoadDataWithNullMeasures extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("DROP TABLE IF EXISTS carbontable")
sql(
"CREATE TABLE carbontable (empno int, empname String, designation String, doj String, " +
"workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
index 7ab1a33..0e333dd 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestDataWithDicExcludeAndInclude.scala
@@ -40,14 +40,22 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
}
def dropTable() = {
- sql("DROP TABLE IF EXISTS t3")
+ sql("DROP TABLE IF EXISTS exclude_include_t3")
+ sql("DROP TABLE IF EXISTS exclude_include_hive_t3")
}
def buildTable() = {
try {
sql(
"""
- CREATE TABLE IF NOT EXISTS t3
+ CREATE TABLE exclude_include_hive_t3
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ row format delimited fields terminated by ','
+ """)
+ sql(
+ """
+ CREATE TABLE exclude_include_t3
(ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'org.apache.carbondata.format'
@@ -65,7 +73,11 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql(
s"""
- LOAD DATA LOCAL INPATH '$filePath' into table t3
+ LOAD DATA LOCAL INPATH '$filePath' into table exclude_include_t3
+ """)
+ sql(
+ s"""
+ LOAD DATA LOCAL INPATH './src/test/resources/emptyDimensionDataHive.csv' into table exclude_include_hive_t3
""")
} catch {
case ex: Throwable => logError(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -81,9 +93,7 @@ class TestLoadDataWithDictionaryExcludeAndInclude extends QueryTest with BeforeA
test("test load data with dictionary exclude & include and with empty dimension") {
checkAnswer(
- sql("select ID from t3"), Seq(Row(1), Row(2), Row(3), Row(4), Row(5), Row(6), Row(7),
- Row(8), Row(9), Row(10), Row(11), Row(12), Row(13), Row(14), Row(15), Row(16), Row
- (17), Row(18), Row(19), Row(20))
+ sql("select ID from exclude_include_t3"), sql("select ID from exclude_include_hive_t3")
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 173fe6b..731ed89 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -529,25 +529,38 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
}
test("test data which contain column with decimal data type in array of struct."){
- sql("DROP TABLE IF EXISTS t3")
+ sql("DROP TABLE IF EXISTS complex_t3")
+ sql("DROP TABLE IF EXISTS complex_hive_t3")
sql(
"""
- CREATE TABLE IF NOT EXISTS t3
+ CREATE TABLE complex_t3
(ID decimal, date Timestamp, country String,
name String, phonetype String, serialname String, salary Int, complex
array<struct<a:decimal(4,2),str:string>>)
STORED BY 'org.apache.carbondata.format'
"""
)
+ sql(
+ """
+ CREATE TABLE complex_hive_t3
+ (ID decimal, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int, complex
+ array<struct<a:decimal(4,2),str:string>>)
+ row format delimited fields terminated by ','
+ """
+ )
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql(s"""
- LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into table t3
+ LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNested.csv' into table complex_t3
""")
- checkAnswer(sql("select count(*) from t3"),Seq(Row(8)))
- checkAnswer(sql("select id from t3 where salary = 15000"),Seq(Row(1)))
+ sql(s"""
+ LOAD DATA LOCAL INPATH './src/test/resources/complexTypeDecimalNestedHive.csv' into table complex_hive_t3
+ """)
+ checkAnswer(sql("select count(*) from complex_t3"),sql("select count(*) from complex_hive_t3"))
+ checkAnswer(sql("select id from complex_t3 where salary = 15000"),sql("select id from complex_hive_t3 where salary = 15000"))
}
test("test data loading when delimiter is '|' and data with header") {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
index 7ec5e9a..9e9fa90 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
@@ -31,6 +31,8 @@ import org.scalatest.BeforeAndAfterAll
class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("DROP TABLE IF EXISTS carbonTable")
+ sql("DROP TABLE IF EXISTS hiveTable")
sql("CREATE TABLE carbonTable (imei String, age Int, num BigInt) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='age,num')")
sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegtiveNumber.csv' INTO TABLE carbonTable")
sql("CREATE TABLE hiveTable (imei String, age Int, num BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
@@ -52,7 +54,7 @@ class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("DROP TABLE carbonTable")
- sql("DROP TABLE hiveTable")
+ sql("DROP TABLE IF EXISTS carbonTable")
+ sql("DROP TABLE IF EXISTS hiveTable")
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b82a960b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
index 2449f64..d81974d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/NullMeasureValueTestCaseFilter.scala
@@ -10,8 +10,9 @@ import org.scalatest.BeforeAndAfterAll
class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ sql("drop table if exists t3")
sql(
- "CREATE TABLE IF NOT EXISTS t3 (ID Int, date Timestamp, country String, name String, " +
+ "CREATE TABLE t3 (ID bigInt, date Timestamp, country String, name String, " +
"phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata.format'"
)
CarbonProperties.getInstance()
@@ -22,13 +23,13 @@ class NullMeasureValueTestCaseFilter extends QueryTest with BeforeAndAfterAll {
test("select ID from t3 where salary is not null") {
checkAnswer(
sql("select ID from t3 where salary is not null"),
- Seq(Row(1.0),Row(4.0)))
+ Seq(Row(1),Row(4)))
}
test("select ID from t3 where salary is null") {
checkAnswer(
sql("select ID from t3 where salary is null"),
- Seq(Row(2.0),Row(3.0)))
+ Seq(Row(2),Row(3)))
}
override def afterAll {
[36/47] incubator-carbondata git commit: [CARBONDATA-118][Bug] Temp
location clean up in compaction (#883)
Posted by ra...@apache.org.
[CARBONDATA-118][Bug] Temp location clean up in compaction (#883)
Passed compaction temporary location to compaction and data load merge flow. Handled clean up of that temp folder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5341c7dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5341c7dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5341c7dc
Branch: refs/heads/master
Commit: 5341c7dce8ce1a260d1ac48af67d58cd12331ba6
Parents: 61b6074
Author: ravikiran23 <ra...@gmail.com>
Authored: Fri Jul 29 06:05:39 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jul 29 06:05:39 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 5 +++
.../carbondata/spark/load/CarbonLoaderUtil.java | 26 +++++++-----
.../spark/rdd/CarbonDataLoadRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 42 ++++++++++++++++----
.../processing/mdkeygen/MDKeyGenStep.java | 2 +-
.../sortandgroupby/sortdata/SortDataRows.java | 2 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 2 +-
.../FileStoreSurrogateKeyGenForCSV.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 7 +++-
9 files changed, 69 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 5852241..86d87eb 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -849,6 +849,11 @@ public final class CarbonCommonConstants {
*/
public static String LEVEL2_COMPACTION_INDEX = ".2";
+ /**
+ * Indicates compaction
+ */
+ public static String COMPACTION_KEY_WORD = "COMPACTION";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index 3ecdfe4..bce99a7 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -428,7 +428,7 @@ public final class CarbonLoaderUtil {
* @param segmentName
*/
public static void deleteLocalDataLoadFolderLocation(CarbonLoadModel loadModel,
- String segmentName) {
+ String segmentName, boolean isCompactionFlow) {
String databaseName = loadModel.getDatabaseName();
String tableName = loadModel.getTableName();
CarbonTableIdentifier carbonTableIdentifier =
@@ -436,26 +436,32 @@ public final class CarbonLoaderUtil {
String segmentId = segmentName.substring(CarbonCommonConstants.LOAD_FOLDER.length());
String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
+ CarbonCommonConstants.UNDERSCORE + loadModel.getTaskNo();
+ if (isCompactionFlow) {
+ tempLocationKey = CarbonCommonConstants.COMPACTION_KEY_WORD + '_' + tempLocationKey;
+ }
// form local store location
String localStoreLocation = getStoreLocation(CarbonProperties.getInstance()
.getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL),
carbonTableIdentifier, segmentId, loadModel.getPartitionId());
try {
CarbonUtil.deleteFoldersAndFiles(new File[] { new File(localStoreLocation) });
+ LOGGER.info("Deleted the local store location" + localStoreLocation);
} catch (CarbonUtilException e) {
LOGGER.error(e, "Failed to delete local data load folder location");
}
// delete ktr file.
- String graphPath = CarbonProperties.getInstance()
- .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL)
- + File.separator + "/etl" + File.separator + databaseName + File.separator + tableName
- + File.separator + loadModel.getSegmentId() + File.separator + loadModel.getTaskNo()
- + File.separator + tableName + ".ktr";
- File path = new File(graphPath);
- if (path.exists()) {
- if (!path.delete()) {
- LOGGER.error("failed to delete the ktr file in path " + path);
+ if (!isCompactionFlow) {
+ String graphPath = CarbonProperties.getInstance()
+ .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL)
+ + File.separator + "/etl" + File.separator + databaseName + File.separator + tableName
+ + File.separator + loadModel.getSegmentId() + File.separator + loadModel.getTaskNo()
+ + File.separator + tableName + ".ktr";
+ File path = new File(graphPath);
+ if (path.exists()) {
+ if (!path.delete()) {
+ LOGGER.error("failed to delete the ktr file in path " + path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 87f7885..36a1c0f 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -208,7 +208,9 @@ class CarbonDataLoadRDD[K, V](
// delete temp location data
val newSlice = CarbonCommonConstants.LOAD_FOLDER + loadCount
try {
- CarbonLoaderUtil.deleteLocalDataLoadFolderLocation(model, newSlice)
+ val isCompaction = false
+ CarbonLoaderUtil
+ .deleteLocalDataLoadFolderLocation(model, newSlice, isCompaction)
} catch {
case e: Exception =>
LOGGER.error(e)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
index 254d51b..7b15cbf 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -21,6 +21,7 @@ import java.util
import java.util.{Collections, List}
import scala.collection.JavaConverters._
+import scala.util.Random
import org.apache.hadoop.mapreduce.Job
import org.apache.spark._
@@ -59,7 +60,7 @@ class CarbonMergerRDD[K, V](
sc.setLocalProperty("spark.scheduler.pool", "DDL")
sc.setLocalProperty("spark.job.interruptOnCancel", "true")
- val storeLocation = carbonMergerMapping.storeLocation
+ var storeLocation: String = null
val hdfsStoreLocation = carbonMergerMapping.hdfsStoreLocation
val metadataFilePath = carbonMergerMapping.metadataFilePath
val mergedLoadName = carbonMergerMapping.mergedLoadName
@@ -70,16 +71,28 @@ class CarbonMergerRDD[K, V](
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
+ carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
+ val tempLocationKey: String = CarbonCommonConstants
+ .COMPACTION_KEY_WORD + '_' + carbonLoadModel
+ .getDatabaseName + '_' + carbonLoadModel
+ .getTableName + '_' + carbonLoadModel.getTaskNo
+
+ val storeLocations = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
+ if (null != storeLocations && storeLocations.length > 0) {
+ storeLocation = storeLocations(Random.nextInt(storeLocations.length))
+ }
+ if (storeLocation == null) {
+ storeLocation = System.getProperty("java.io.tmpdir")
+ }
+ storeLocation = storeLocation + '/' + System.nanoTime() + '/' + theSplit.index
+ CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
+ LOGGER.info("Temp storeLocation taken is " + storeLocation)
var mergeStatus = false
+ var mergeNumber = ""
try {
var dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
- val tempLocationKey: String = carbonLoadModel.getDatabaseName + '_' + carbonLoadModel
- .getTableName + carbonLoadModel.getTaskNo
- CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation)
-
// sorting the table block info List.
var tableBlockInfoList = carbonSparkPartition.tableBlockInfos
@@ -126,7 +139,7 @@ class CarbonMergerRDD[K, V](
}
}
- val mergeNumber = mergedLoadName
+ mergeNumber = mergedLoadName
.substring(mergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) +
CarbonCommonConstants.LOAD_FOLDER.length(), mergedLoadName.length()
)
@@ -135,7 +148,8 @@ class CarbonMergerRDD[K, V](
factTableName,
carbonLoadModel.getTaskNo,
"0",
- mergeNumber
+ mergeNumber,
+ true
)
carbonLoadModel.setSegmentId(mergeNumber)
@@ -157,6 +171,18 @@ class CarbonMergerRDD[K, V](
LOGGER.error(e)
throw e
}
+ finally {
+ // delete temp location data
+ val newSlice = CarbonCommonConstants.LOAD_FOLDER + mergeNumber
+ try {
+ val isCompactionFlow = true
+ CarbonLoaderUtil
+ .deleteLocalDataLoadFolderLocation(carbonLoadModel, newSlice, isCompactionFlow)
+ } catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ }
+ }
var finished = false
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java b/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
index 8ddb429..6b997d5 100644
--- a/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
@@ -259,7 +259,7 @@ public class MDKeyGenStep extends BaseStep {
this.tableName = meta.getTableName();
storeLocation = CarbonDataProcessorUtil
.getLocalDataFolderLocation(meta.getSchemaName(), meta.getTableName(),
- String.valueOf(meta.getTaskNo()), meta.getPartitionID(), meta.getSegmentId()+"");
+ String.valueOf(meta.getTaskNo()), meta.getPartitionID(), meta.getSegmentId()+"", false);
isNoDictionaryDimension =
RemoveDictionaryUtil.convertStringToBooleanArr(meta.getNoDictionaryDimsMapping());
isUseInvertedIndex =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index 1b3de04..bb8ab2a 100644
--- a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -525,7 +525,7 @@ public class SortDataRows {
private void updateSortTempFileLocation() {
String carbonDataDirectoryPath = CarbonDataProcessorUtil
.getLocalDataFolderLocation(schemaName, tableName, taskNo, partitionID,
- segmentId);
+ segmentId, false);
this.tempFileLocation =
carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
LOGGER.info("temp file location" + this.tempFileLocation);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index d5ecb37..c2093ad 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -707,7 +707,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
private void updateStoreLocation() {
loadFolderLoc = CarbonDataProcessorUtil
.getLocalDataFolderLocation(meta.getSchemaName(), meta.getTableName(), meta.getTaskNo(),
- meta.getPartitionID(), meta.getSegmentId()+"");
+ meta.getPartitionID(), meta.getSegmentId()+"", false);
}
private String getBadLogStoreLocation(String storeLocation) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index a920d63..b4822db 100644
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -193,7 +193,8 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
private String checkAndCreateLoadFolderNumber(String databaseName,
String tableName) throws KettleException {
String carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId+"");
+ .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId + "",
+ false);
boolean isDirCreated = new File(carbonDataDirectoryPath).mkdirs();
if (!isDirCreated) {
throw new KettleException("Unable to create data load directory" + carbonDataDirectoryPath);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5341c7dc/processing/src/main/java/org/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/carbondata/processing/util/CarbonDataProcessorUtil.java
index 8f2c9e7..84274ed 100644
--- a/processing/src/main/java/org/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -262,9 +262,14 @@ public final class CarbonDataProcessorUtil {
* @return
*/
public static String getLocalDataFolderLocation(String databaseName, String tableName,
- String taskId, String partitionId, String segmentId) {
+ String taskId, String partitionId, String segmentId, boolean isCompactionFlow) {
String tempLocationKey = databaseName + CarbonCommonConstants.UNDERSCORE + tableName
+ CarbonCommonConstants.UNDERSCORE + taskId;
+ if(isCompactionFlow){
+ tempLocationKey = CarbonCommonConstants
+ .COMPACTION_KEY_WORD + '_' + tempLocationKey;
+ }
+
String baseStorePath = CarbonProperties.getInstance()
.getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
CarbonTable carbonTable = CarbonMetadata.getInstance()
[32/47] incubator-carbondata git commit: [CARBONDATA-110] delete
segment selected for compaction. (#874)
Posted by ra...@apache.org.
[CARBONDATA-110] delete segment selected for compaction. (#874)
if the compaction is running and the user deleted the segment which is under compaction.
then not updating the table status, and printing the error message in compaction.
if the compaction is running and the user deleted the segment which is under compaction.
then not updating the table status, and printing the error message in compaction.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bf7f9f7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bf7f9f7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bf7f9f7d
Branch: refs/heads/master
Commit: bf7f9f7d15c430adc629508d6a5fed1d8edbc7b7
Parents: 973def9
Author: ravikiran23 <ra...@gmail.com>
Authored: Thu Jul 28 21:41:13 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Thu Jul 28 21:41:13 2016 +0530
----------------------------------------------------------------------
.../spark/merger/CarbonDataMergerUtil.java | 8 +++++
.../org/carbondata/spark/rdd/Compactor.scala | 31 +++++++++++++-------
2 files changed, 29 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf7f9f7d/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
index f033993..b290dd5 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -161,6 +161,14 @@ public final class CarbonDataMergerUtil {
for (LoadMetadataDetails loadDetail : loadDetails) {
// check if this segment is merged.
if (loadsToMerge.contains(loadDetail)) {
+ // if the compacted load is deleted after the start of the compaction process,
+ // then need to discard the compaction process and treat it as failed compaction.
+ if (loadDetail.getLoadStatus()
+ .equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
+ LOGGER.error("Compaction is aborted as the segment " + loadDetail.getLoadName()
+ + " is deleted after the compaction is started.");
+ return tableStatusUpdationStatus;
+ }
loadDetail.setLoadStatus(CarbonCommonConstants.SEGMENT_COMPACTED);
loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
loadDetail.setMergedLoadName(mergedLoadNumber);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bf7f9f7d/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
index 7e19ded..2b17d57 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
@@ -115,18 +115,29 @@ object Compactor {
if (finalMergeStatus) {
val endTime = System.nanoTime();
logger.info("time taken to merge " + mergedLoadName + " is " + (endTime - startTime))
- CarbonDataMergerUtil
+ if (!CarbonDataMergerUtil
.updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath(),
mergedLoadName, carbonLoadModel, mergeLoadStartTime, compactionType
- )
- logger
- .audit("Compaction request completed for table " + carbonLoadModel
- .getDatabaseName + "." + carbonLoadModel.getTableName
- )
- logger
- .info("Compaction request completed for table " + carbonLoadModel
- .getDatabaseName + "." + carbonLoadModel.getTableName
- )
+ )) {
+ logger
+ .audit("Compaction request failed for table " + carbonLoadModel
+ .getDatabaseName + "." + carbonLoadModel.getTableName
+ )
+ logger
+ .error("Compaction request failed for table " + carbonLoadModel
+ .getDatabaseName + "." + carbonLoadModel.getTableName
+ )
+ }
+ else {
+ logger
+ .audit("Compaction request completed for table " + carbonLoadModel
+ .getDatabaseName + "." + carbonLoadModel.getTableName
+ )
+ logger
+ .info("Compaction request completed for table " + carbonLoadModel
+ .getDatabaseName + "." + carbonLoadModel.getTableName
+ )
+ }
}
else {
logger
[40/47] incubator-carbondata git commit: [bug]fix bug of duplicate
rows in UnivocityCsvParser (#877)
Posted by ra...@apache.org.
[bug]fix bug of duplicate rows in UnivocityCsvParser (#877)
* fix bug of duplicate rows in UnivocityCsvParser
* test 10 million
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/87f0deca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/87f0deca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/87f0deca
Branch: refs/heads/master
Commit: 87f0deca35981e890b2d89565009dbde3bce4610
Parents: 2d50d5c
Author: david <qi...@qq.com>
Authored: Fri Jul 29 17:42:56 2016 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Fri Jul 29 15:12:56 2016 +0530
----------------------------------------------------------------------
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../dataload/TestLoadDataWithJunkChars.scala | 43 +++++
.../csvreaderstep/CustomDataStream.java | 108 +++++++++++++
.../processing/csvreaderstep/CustomReader.java | 157 -------------------
.../csvreaderstep/UnivocityCsvParser.java | 13 +-
5 files changed, 156 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/87f0deca/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index ff7e360..0e15803 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -297,8 +297,8 @@ object GlobalDictionaryUtil extends Logging {
val dictFilePaths = dictDetail.dictFilePaths
val dictFileExists = dictDetail.dictFileExists
val columnIdentifier = dictDetail.columnIdentifiers
- val hdfstemplocation = CarbonProperties.getInstance.
- getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION)
+ val hdfstemplocation = CarbonProperties.getInstance.getProperty(
+ CarbonCommonConstants.HDFS_TEMP_LOCATION, System.getProperty("java.io.tmpdir"))
val lockType = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/87f0deca/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithJunkChars.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithJunkChars.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithJunkChars.scala
new file mode 100644
index 0000000..5472e7a
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithJunkChars.scala
@@ -0,0 +1,43 @@
+package org.carbondata.integration.spark.testsuite.dataload;
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+
+import org.scalatest.BeforeAndAfterAll
+import java.io.File
+import java.io.BufferedWriter
+import java.io.FileWriter
+import java.util.Random
+import org.apache.spark.sql.Row
+
+class TestLoadDataWithJunkChars extends QueryTest with BeforeAndAfterAll {
+ var filePath = ""
+ val junkchars = "\u01cd\u01ce\u01cf\u01d0\u01d1\u01d2\u01d3\u01d4\u01d5\u01d6\u01d7\u01d8\u01d9\u01da\u01db\u01dc\u01dd\u01de\u01df\u01e0\u01e1\u01e2\u01e3\u01e4\u01e5\u01e6\u01e7\u01e8\u01e9\u01ea\u01eb\u01ec\u01ed\u01ee\u01ef\u01f0"
+
+ def buildTestData() = {
+ val pwd = new File(this.getClass.getResource("/").getPath + "/../../").getCanonicalPath
+ filePath = pwd + "/target/junkcharsdata.csv"
+ val file = new File(filePath)
+ val writer = new BufferedWriter(new FileWriter(file))
+ writer.write("c1,c2\n")
+ val random = new Random
+ for (i <- 1 until 1000000) {
+ writer.write("a" + i + "," + junkchars + "\n")
+ }
+ writer.write("a1000000," + junkchars)
+ writer.close
+ }
+
+ test("[bug]fix bug of duplicate rows in UnivocityCsvParser #877") {
+ buildTestData()
+ sql("drop table if exists junkcharsdata")
+ sql("""create table if not exists junkcharsdata
+ (c1 string, c2 string)
+ STORED BY 'org.apache.carbondata.format'""")
+ sql(s"LOAD DATA LOCAL INPATH '$filePath' into table junkcharsdata").show
+ sql("select * from junkcharsdata").show(20,false)
+ checkAnswer(sql("select count(*) from junkcharsdata"), Seq(Row(1000000)))
+ sql("drop table if exists junkcharsdata")
+ new File(filePath).delete()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/87f0deca/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomDataStream.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomDataStream.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomDataStream.java
new file mode 100644
index 0000000..9c18eba
--- /dev/null
+++ b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomDataStream.java
@@ -0,0 +1,108 @@
+package org.carbondata.processing.csvreaderstep;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Custom reader class to read the data from file it will take care of reading
+ * till the limit assigned to this class
+ */
+public class CustomDataStream extends InputStream {
+
+ /**
+ * byte value of the new line character
+ */
+ private static final byte END_OF_LINE_BYTE_VALUE = '\n';
+
+ /**
+ * number of extra character to read
+ */
+ private static final int NUMBER_OF_EXTRA_CHARACTER_TO_READ = 100;
+
+ /**
+ * number of bytes remaining
+ */
+ private long remaining;
+ /**
+ * to check whether end of line is found
+ */
+ private boolean endOfLineFound = false;
+
+ private DataInputStream in;
+
+ public CustomDataStream(DataInputStream in, long limit) {
+ this.in = in;
+ this.remaining = limit;
+ }
+
+ /**
+ * Below method will be used to read the data from file
+ *
+ * @throws IOException
+ * problem while reading
+ */
+ @Override
+ public int read() throws IOException {
+ if (this.remaining == 0) {
+ return -1;
+ } else {
+ int var1 = this.in.read();
+ if (var1 >= 0) {
+ --this.remaining;
+ }
+
+ return var1;
+ }
+ }
+
+ /**
+ * Below method will be used to read the data from file. If limit reaches in
+ * that case it will read until new line character is reached
+ *
+ * @param buffer
+ * buffer in which data will be read
+ * @param offset
+ * from position to buffer will be filled
+ * @param length
+ * number of character to be read
+ * @throws IOException
+ * problem while reading
+ */
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ if (this.remaining == 0) {
+ return -1;
+ } else {
+ if (this.remaining < length) {
+ length = (int) this.remaining;
+ }
+
+ length = this.in.read(buffer, offset, length);
+ if (length >= 0) {
+ this.remaining -= length;
+ if (this.remaining == 0 && !endOfLineFound) {
+ endOfLineFound = true;
+ this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
+ } else if (endOfLineFound) {
+ int end = offset + length;
+ for (int i = offset; i < end; i++) {
+ if (buffer[i] == END_OF_LINE_BYTE_VALUE) {
+ this.remaining = 0;
+ return (i - offset) + 1;
+ }
+ }
+ this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
+ }
+ }
+ return length;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (in != null) {
+ in.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/87f0deca/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomReader.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomReader.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomReader.java
deleted file mode 100644
index 49c5b15..0000000
--- a/processing/src/main/java/org/carbondata/processing/csvreaderstep/CustomReader.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.processing.csvreaderstep;
-
-import java.io.IOException;
-import java.io.Reader;
-
-/**
- * Custom reader class to read the data from file
- * it will take care of reading till the limit assigned to this
- * class
- */
-public final class CustomReader extends Reader {
-
- /**
- * byte value of the new line character
- */
- private static final byte END_OF_LINE_BYTE_VALUE = '\n';
-
- /**
- * number of extra character to read
- */
- private static final int NUMBER_OF_EXTRA_CHARACTER_TO_READ = 100;
-
- /**
- * number of bytes remaining
- */
- private long remaining;
-
- /**
- * reader
- */
- private Reader reader;
-
- /**
- * to check whether end of line is found
- */
- private boolean endOfLineFound = false;
-
- public CustomReader(Reader var1) {
- this.reader = var1;
- }
-
- /**
- * Below method will be used to read the data from file
- *
- * @throws IOException problem while reading
- */
- public int read() throws IOException {
- if (this.remaining == 0) {
- return -1;
- } else {
- int var1 = this.reader.read();
- if (var1 >= 0) {
- --this.remaining;
- }
-
- return var1;
- }
- }
-
- /**
- * Below method will be used to read the data from file.
- * If limit reaches in that case it will read until new line character is reached
- *
- * @param buffer buffer in which data will be read
- * @param offset from position to buffer will be filled
- * @param length number of character to be read
- * @throws IOException problem while reading
- */
- public int read(char[] buffer, int offset, int length) throws IOException {
- if (this.remaining == 0) {
- return -1;
- } else {
- if (this.remaining < length) {
- length = (int) this.remaining;
- }
-
- length = this.reader.read(buffer, offset, length);
- if (length >= 0) {
- this.remaining -= length;
- if (this.remaining == 0 && !endOfLineFound) {
- if (buffer[length - 1] != END_OF_LINE_BYTE_VALUE) {
- endOfLineFound = true;
- this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
- }
- } else if (endOfLineFound) {
- for (int i = 0; i < length; i++) {
- if (buffer[i] == END_OF_LINE_BYTE_VALUE) {
- this.remaining = 0;
- return i + 1;
- }
- }
- this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
- }
- }
- return length;
- }
- }
-
- /**
- * number of bytes to skip
- */
- public long skip(long numberOfCharacterToSkip) throws IOException {
- throw new UnsupportedOperationException("Not supported");
- }
-
- /**
- * to set the stream position
- */
- public void mark(int readAheadLimit) throws IOException {
- throw new UnsupportedOperationException("Not supported");
- }
-
- /**
- * to close the stream
- */
- public void close() throws IOException {
- this.reader.close();
- }
-
- /**
- * Max number of bytes that can be read by the stream
- *
- * @param limit
- */
- public void setLimit(long limit) {
- this.remaining = limit;
- }
-
- /**
- * number of remaining bytes
- *
- * @return number of remaining bytes
- */
- public final long getRemaining() {
- return this.remaining;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/87f0deca/processing/src/main/java/org/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvreaderstep/UnivocityCsvParser.java b/processing/src/main/java/org/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
index 8359576..9594a24 100644
--- a/processing/src/main/java/org/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
+++ b/processing/src/main/java/org/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
@@ -119,12 +119,7 @@ public class UnivocityCsvParser {
// calculate the end offset the block
long endOffset =
this.csvParserVo.getBlockDetailsList().get(blockCounter).getBlockLength() + startOffset;
- // if start offset is not 0 then we need to set the offset to the start of a line
- // so if offset is not zero we are setting to -1 so to check if current position itself is
- // start of the block so -1 will ensure whether last character is new line character or not
- if (startOffset != 0) {
- startOffset -= 1;
- }
+
// create a input stream for the block
DataInputStream dataInputStream = FileFactory
.getDataInputStream(this.csvParserVo.getBlockDetailsList().get(blockCounter).getFilePath(),
@@ -134,10 +129,8 @@ public class UnivocityCsvParser {
LineReader lineReader = new LineReader(dataInputStream, 1);
startOffset += lineReader.readLine(new Text(), 0);
}
- CustomReader reader =
- new CustomReader(new BufferedReader(new InputStreamReader(dataInputStream)));
- reader.setLimit(endOffset - startOffset);
- inputStreamReader = reader;
+ inputStreamReader = new BufferedReader(new InputStreamReader(
+ new CustomDataStream(dataInputStream, endOffset - startOffset)));
}
/**
[02/47] incubator-carbondata git commit: [CARBONDATA-91] Concurrent
query returning empty result (#849)
Posted by ra...@apache.org.
[CARBONDATA-91] Concurrent query returning empty result (#849)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a5714ace
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a5714ace
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a5714ace
Branch: refs/heads/master
Commit: a5714acedb453fcf148222854177f383aa88eaab
Parents: a326bb8
Author: Mohammad Shahid Khan <mo...@gmail.com>
Authored: Thu Jul 21 20:00:46 2016 +0530
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Thu Jul 21 20:00:46 2016 +0530
----------------------------------------------------------------------
.../carbondata/core/carbon/datastore/SegmentTaskIndexStore.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a5714ace/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
index 28a892e..c94a100 100644
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
@@ -143,7 +143,6 @@ public class SegmentTaskIndexStore {
if (null == taskIdToSegmentIndexMap) {
// creating a map of take if to table segment
taskIdToSegmentIndexMap = new HashMap<String, AbstractIndex>();
- tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
Iterator<Entry<String, List<TableBlockInfo>>> iterator =
taskIdToTableBlockInfoMap.entrySet().iterator();
while (iterator.hasNext()) {
@@ -152,6 +151,7 @@ public class SegmentTaskIndexStore {
taskIdToSegmentIndexMap.put(taskId,
loadBlocks(taskId, taskToBlockInfoList.getValue(), absoluteTableIdentifier));
}
+ tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
// removing from segment lock map as once segment is loaded
//if concurrent query is coming for same segment
// it will wait on the lock so after this segment will be already
[10/47] incubator-carbondata git commit: [CARBONDATA-93][Bug] Task
not re-submitted by spark on data load failure (#850)
Posted by ra...@apache.org.
[CARBONDATA-93][Bug] Task not re-submitted by spark on data load failure (#850)
In case of data load failure on any executor, exception thrown is caught and logged as an error but the exception is not re-thrown to the spark therefore spark does not resubmits the task to the available executors again.
Fix: Catch the exception, log the error and after that re-throw the exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/30e4f259
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/30e4f259
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/30e4f259
Branch: refs/heads/master
Commit: 30e4f25942f9c40ea4895410bdc25b679f6322dc
Parents: a81ed98
Author: manishgupta88 <to...@gmail.com>
Authored: Sat Jul 23 12:34:33 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 12:34:33 2016 +0530
----------------------------------------------------------------------
.../spark/rdd/CarbonDataLoadRDD.scala | 16 +---
.../spark/rdd/CarbonDataRDDFactory.scala | 99 +++++++++++---------
2 files changed, 59 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/30e4f259/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 06bd12f..18ea3b0 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -201,12 +201,10 @@ class CarbonDataLoadRDD[K, V](
dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
logInfo("Bad Record Found")
} else {
- dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- LOGGER.error(e)
+ throw e
}
case e: Exception =>
- dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- LOGGER.error(e)
+ throw e
} finally {
if (!CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(dataloadStatus)) {
val newSlice = CarbonCommonConstants.LOAD_FOLDER + loadCount
@@ -216,10 +214,7 @@ class CarbonDataLoadRDD[K, V](
case e: Exception =>
LOGGER.error(e)
}
- dataloadStatus = checkAndLoadAggregationTable
- if (CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(dataloadStatus)) {
- logInfo("DataLoad failure")
- } else if (CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
+ if (CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
.equals(dataloadStatus)) {
logInfo("DataLoad complete")
logInfo("Data Load partially successful with LoadCount:" + loadCount)
@@ -229,15 +224,14 @@ class CarbonDataLoadRDD[K, V](
CarbonTimeStatisticsFactory.getLoadStatisticsInstance.printStatisticsInfo(
model.getPartitionId)
}
- } else {
- logInfo("DataLoad failure")
}
}
}
} catch {
case e: Exception =>
- dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
logInfo("DataLoad failure")
+ LOGGER.error(e)
+ throw e
}
def setModelAndBlocksInfo(): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/30e4f259/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7271ee0..b314f88 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -766,50 +766,59 @@ object CarbonDataRDDFactory extends Logging {
CarbonLoaderUtil.checkAndCreateCarbonDataLocation(hdfsStoreLocation,
carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName,
partitioner.partitionCount, currentLoadCount.toString)
- val status = new
- CarbonDataLoadRDD(sc.sparkContext,
- new DataLoadResultImpl(),
- carbonLoadModel,
- storeLocation,
- hdfsStoreLocation,
- kettleHomePath,
- partitioner,
- columinar,
- currentRestructNumber,
- currentLoadCount,
- cubeCreationTime,
- schemaLastUpdatedTime,
- blocksGroupBy,
- isTableSplitPartition
- ).collect()
- val newStatusMap = scala.collection.mutable.Map.empty[String, String]
- status.foreach { eachLoadStatus =>
- val state = newStatusMap.get(eachLoadStatus._1)
- state match {
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
- if eachLoadStatus._2.getLoadStatus == CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- case _ =>
- newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
- }
- }
-
var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
- newStatusMap.foreach {
- case (key, value) =>
- if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
- } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
- !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
- loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
+ var status: Array[(String, LoadMetadataDetails)] = null
+ try {
+ status = new
+ CarbonDataLoadRDD(sc.sparkContext,
+ new DataLoadResultImpl(),
+ carbonLoadModel,
+ storeLocation,
+ hdfsStoreLocation,
+ kettleHomePath,
+ partitioner,
+ columinar,
+ currentRestructNumber,
+ currentLoadCount,
+ cubeCreationTime,
+ schemaLastUpdatedTime,
+ blocksGroupBy,
+ isTableSplitPartition
+ ).collect()
+ val newStatusMap = scala.collection.mutable.Map.empty[String, String]
+ status.foreach { eachLoadStatus =>
+ val state = newStatusMap.get(eachLoadStatus._1)
+ state match {
+ case Some(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
+ case Some(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
+ if eachLoadStatus._2.getLoadStatus ==
+ CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
+ case _ =>
+ newStatusMap.put(eachLoadStatus._1, eachLoadStatus._2.getLoadStatus)
}
- }
+ }
- if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
- partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
- loadStatus = partitionStatus
+ newStatusMap.foreach {
+ case (key, value) =>
+ if (value == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ } else if (value == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS &&
+ !loadStatus.equals(CarbonCommonConstants.STORE_LOADSTATUS_FAILURE)) {
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
+ }
+ }
+
+ if (loadStatus != CarbonCommonConstants.STORE_LOADSTATUS_FAILURE &&
+ partitionStatus == CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) {
+ loadStatus = partitionStatus
+ }
+ } catch {
+ case ex: Throwable =>
+ loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_FAILURE
+ logInfo("DataLoad failure")
+ logger.error(ex)
}
if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
@@ -823,12 +832,12 @@ object CarbonDataRDDFactory extends Logging {
)
message = "Aggregate table creation failure"
} else {
- val (result, _) = status(0)
- val newSlice = CarbonCommonConstants.LOAD_FOLDER + result
CarbonLoaderUtil.deleteSegment(carbonLoadModel, currentLoadCount)
val aggTables = carbonTable.getAggregateTablesName
if (null != aggTables && !aggTables.isEmpty) {
// TODO:need to clean aggTable
+ val (result, _) = status(0)
+ val newSlice = CarbonCommonConstants.LOAD_FOLDER + result
aggTables.asScala.foreach { aggTableName =>
CarbonLoaderUtil
.deleteSlice(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
@@ -837,12 +846,12 @@ object CarbonDataRDDFactory extends Logging {
)
}
}
- message = "Dataload failure"
+ message = "DataLoad failure"
}
logInfo("********clean up done**********")
logger.audit(s"Data load is failed for " +
s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
- logWarning("Unable to write load metadata file")
+ logWarning("Cannot write load metadata file as data load failed")
throw new Exception(message)
} else {
val metadataDetails = status(0)._2
[20/47] incubator-carbondata git commit: [CARBONDATA-87][Bug]
deleting the temp files in case of success / failure of load. (#846)
Posted by ra...@apache.org.
[CARBONDATA-87][Bug] deleting the temp files in case of success / failure of load. (#846)
deleting the temp folder location in case of success / failure of load.
deleting the ktr file from temp location.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/577764be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/577764be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/577764be
Branch: refs/heads/master
Commit: 577764bef23c005b3599761e397ffc2110325c36
Parents: bb8b0fd
Author: ravikiran23 <ra...@gmail.com>
Authored: Mon Jul 25 15:43:25 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 15:43:25 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/spark/load/CarbonLoaderUtil.java | 14 ++++++++++++++
.../org/carbondata/spark/rdd/CarbonDataLoadRDD.scala | 15 ++++++++-------
2 files changed, 22 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/577764be/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index 3d17141..3ecdfe4 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -445,6 +445,20 @@ public final class CarbonLoaderUtil {
} catch (CarbonUtilException e) {
LOGGER.error(e, "Failed to delete local data load folder location");
}
+
+ // delete ktr file.
+ String graphPath = CarbonProperties.getInstance()
+ .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL)
+ + File.separator + "/etl" + File.separator + databaseName + File.separator + tableName
+ + File.separator + loadModel.getSegmentId() + File.separator + loadModel.getTaskNo()
+ + File.separator + tableName + ".ktr";
+ File path = new File(graphPath);
+ if (path.exists()) {
+ if (!path.delete()) {
+ LOGGER.error("failed to delete the ktr file in path " + path);
+ }
+ }
+
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/577764be/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 8869368..60a3a8a 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -206,14 +206,15 @@ class CarbonDataLoadRDD[K, V](
case e: Exception =>
throw e
} finally {
+ // delete temp location data
+ val newSlice = CarbonCommonConstants.LOAD_FOLDER + loadCount
+ try {
+ CarbonLoaderUtil.deleteLocalDataLoadFolderLocation(model, newSlice)
+ } catch {
+ case e: Exception =>
+ LOGGER.error(e)
+ }
if (!CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(dataloadStatus)) {
- val newSlice = CarbonCommonConstants.LOAD_FOLDER + loadCount
- try {
- CarbonLoaderUtil.deleteLocalDataLoadFolderLocation(model, newSlice)
- } catch {
- case e: Exception =>
- LOGGER.error(e)
- }
if (CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS
.equals(dataloadStatus)) {
logInfo("DataLoad complete")
[42/47] incubator-carbondata git commit: Merge remote-tracking branch
'carbon_master/master' into apache/master
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/lcm/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/lcm/locks/ZooKeeperLocking.java
index d0b0453,0000000..82fac7b
mode 100644,000000..100644
--- a/processing/src/main/java/org/carbondata/lcm/locks/ZooKeeperLocking.java
+++ b/processing/src/main/java/org/carbondata/lcm/locks/ZooKeeperLocking.java
@@@ -1,168 -1,0 +1,195 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.lcm.locks;
+
++import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
++import org.carbondata.core.util.CarbonProperties;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * For Handling the zookeeper locking implementation
+ */
+public class ZooKeeperLocking extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
+
+ /**
+ * zk is the zookeeper client instance
+ */
+ private static ZooKeeper zk;
+
+ /**
+ * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
+ * maintained.
+ */
+ private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
+
+ /**
+ * Unique folder for each table with DatabaseName_TableName
+ */
+ private final String tableIdFolder;
+
+ /**
+ * lockName is the name of the lock to use. This name should be same for every process that want
+ * to share the same lock
+ */
+ private String lockName;
+
+ /**
+ * lockPath is the unique path created for the each instance of the carbon lock.
+ */
+ private String lockPath;
+
+ private String lockTypeFolder;
+
++ public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
++ this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
++ lockFile);
++ }
++
+ /**
- * @param tableIdentifier
++ * @param lockLocation
+ * @param lockFile
+ */
- public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
++ public ZooKeeperLocking(String lockLocation, String lockFile) {
+ this.lockName = lockFile;
- this.tableIdFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName();
++ this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
+
- zk = ZookeeperInit.getInstance().getZookeeper();
++ String zooKeeperUrl =
++ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
++ zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
+
- this.lockTypeFolder =
- zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
- + '.' + tableIdentifier.getTableName() + CarbonCommonConstants.FILE_SEPARATOR
- + lockFile;
++ this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
++ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ try {
+ createBaseNode();
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.tableIdFolder, true)) {
- zk.create(this.tableIdFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
++ createRecursivly(this.tableIdFolder);
+ }
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.lockTypeFolder, true)) {
+ zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ initRetry();
+ }
+
+ /**
+ * Creating a znode in which all the znodes (lock files )are maintained.
+ */
+ private void createBaseNode() throws KeeperException, InterruptedException {
+ if (null == zk.exists(zooKeeperLocation, true)) {
+ // creating a znode in which all the znodes (lock files )are maintained.
+ zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
++ * Create zookeepr node if not exist
++ * @param path
++ * @throws KeeperException
++ * @throws InterruptedException
++ */
++ private void createRecursivly(String path) throws KeeperException, InterruptedException {
++ try {
++ if (zk.exists(path, true) == null && path.length() > 0) {
++ String temp = path.substring(0, path.lastIndexOf(File.separator));
++ createRecursivly(temp);
++ zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
++ } else {
++ return;
++ }
++ } catch (KeeperException e) {
++ throw e;
++ } catch (InterruptedException e) {
++ throw e;
++ }
++
++ }
++ /**
+ * Handling of the locking mechanism using zoo keeper.
+ */
+ @Override public boolean lock() {
+ try {
+ // create the lock file with lockName.
+ lockPath =
+ zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
+ Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
+
+ // get the children present in zooKeeperLocation.
+ List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
+
+ // sort the childrens
+ Collections.sort(nodes);
+
+ // here the logic is , for each lock request zookeeper will create a file ending with
+ // incremental digits.
+ // so first request will be 00001 next is 00002 and so on.
+ // if the current request is 00002 and already one previous request(00001) is present then get
+ // children will give both nodes.
+ // after the sort we are checking if the lock path is first or not .if it is first then lock
+ // has been acquired.
+
+ if (lockPath.endsWith(nodes.get(0))) {
+ return true;
+ } else {
+ // if locking failed then deleting the created lock as next time again new lock file will be
+ // created.
+ zk.delete(lockPath, -1);
+ return false;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * @return status where lock file is unlocked or not.
+ */
+ @Override public boolean unlock() {
+ try {
+ // exists will return null if the path doesn't exists.
+ if (null != zk.exists(lockPath, true)) {
+ zk.delete(lockPath, -1);
+ lockPath = null;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
index 6f7a2b1,6b997d5..d69e42c
--- a/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
@@@ -258,8 -258,8 +258,8 @@@ public class MDKeyGenStep extends BaseS
private boolean setStepConfiguration() {
this.tableName = meta.getTableName();
storeLocation = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(meta.getSchemaName(), meta.getTableName(),
+ .getLocalDataFolderLocation(meta.getDatabaseName(), meta.getTableName(),
- String.valueOf(meta.getTaskNo()), meta.getPartitionID(), meta.getSegmentId()+"");
+ String.valueOf(meta.getTaskNo()), meta.getPartitionID(), meta.getSegmentId()+"", false);
isNoDictionaryDimension =
RemoveDictionaryUtil.convertStringToBooleanArr(meta.getNoDictionaryDimsMapping());
isUseInvertedIndex =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index cc2bd17,bb8ab2a..811a8ee
--- a/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@@ -520,8 -524,8 +520,8 @@@ public class SortDataRows
*/
private void updateSortTempFileLocation() {
String carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(schemaName, tableName, taskNo, partitionID,
+ .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID,
- segmentId);
+ segmentId, false);
this.tempFileLocation =
carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
LOGGER.info("temp file location" + this.tempFileLocation);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index b20bcf0,b72de8e..ee6cfd0
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@@ -33,7 -33,9 +33,8 @@@ import java.util.concurrent.locks.Reent
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.cache.dictionary.Dictionary;
+ import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.file.manager.composite.IFileManagerComposite;
import org.carbondata.core.keygenerator.KeyGenException;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --cc processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index fd268ab,c2093ad..003b629
--- a/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@@ -48,8 -48,12 +48,9 @@@ import org.carbondata.core.cache.dictio
import org.carbondata.core.carbon.metadata.CarbonMetadata;
import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.carbon.metadata.schema.table.CarbonTable;
+ import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.file.manager.composite.FileData;
-import org.carbondata.core.file.manager.composite.FileManager;
-import org.carbondata.core.file.manager.composite.IFileManagerComposite;
import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@@ -702,8 -706,8 +704,8 @@@ public class CarbonCSVBasedSeqGenStep e
*/
private void updateStoreLocation() {
loadFolderLoc = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(meta.getSchemaName(), meta.getTableName(), meta.getTaskNo(),
+ .getLocalDataFolderLocation(meta.getDatabaseName(), meta.getTableName(), meta.getTaskNo(),
- meta.getPartitionID(), meta.getSegmentId()+"");
+ meta.getPartitionID(), meta.getSegmentId()+"", false);
}
private String getBadLogStoreLocation(String storeLocation) {
@@@ -1824,5 -1836,45 +1834,45 @@@
}
}
+ private CarbonDimension[] populateNameToCarbonDimensionMap() {
+ CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- meta.getSchemaName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
++ meta.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + meta.getTableName());
+ List<CarbonDimension> dimensionsList = carbonTable.getDimensionByTableName(meta.getTableName());
+ CarbonDimension[] dimensionOrdinalToDimensionMapping =
+ new CarbonDimension[meta.getColumnSchemaDetailsWrapper().getColumnSchemaDetailsMap()
+ .size()];
+ List<CarbonDimension> dimListExcludingNoDictionaryColumn = dimensionsList;
+ if (null != meta.getNoDictionaryDims() && meta.getNoDictionaryDims().length() > 0) {
+ dimListExcludingNoDictionaryColumn =
+ new ArrayList<>(dimensionsList.size() - meta.noDictionaryCols.length);
+ for (CarbonDimension dimension : dimensionsList) {
+ if (!dimension.getEncoder().isEmpty()) {
+ dimListExcludingNoDictionaryColumn.add(dimension);
+ }
+ }
+ }
+ for (int i = 0; i < dimListExcludingNoDictionaryColumn.size(); i++) {
+ CarbonDimension dimension = dimListExcludingNoDictionaryColumn.get(meta.memberMapping[i]);
+ if (dimension.isComplex()) {
+ populateComplexDimension(dimensionOrdinalToDimensionMapping, dimension);
+ } else {
+ dimensionOrdinalToDimensionMapping[meta.memberMapping[i]] = dimension;
+ }
+ }
+ return dimensionOrdinalToDimensionMapping;
+ }
+
+ private void populateComplexDimension(CarbonDimension[] dimensionOrdinalToDimensionMapping,
+ CarbonDimension dimension) {
+ List<CarbonDimension> listOfChildDimensions = dimension.getListOfChildDimensions();
+ for (CarbonDimension childDimension : listOfChildDimensions) {
+ if (childDimension.isComplex()) {
+ populateComplexDimension(dimensionOrdinalToDimensionMapping, childDimension);
+ } else {
+ dimensionOrdinalToDimensionMapping[childDimension.getOrdinal()] = childDimension;
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/processing/src/main/java/org/carbondata/processing/util/CarbonSchemaParser.java
----------------------------------------------------------------------
[09/47] incubator-carbondata git commit: [CARBONDATA-68] Added query
statistic for detail query case measuring carbon executor time (#852)
Posted by ra...@apache.org.
[CARBONDATA-68] Added query statistic for detail query case measuring carbon executor time (#852)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a81ed981
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a81ed981
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a81ed981
Branch: refs/heads/master
Commit: a81ed981230e3fddd8e1c29d774266c171a54643
Parents: 584402e
Author: Kumar Vishal <ku...@gmail.com>
Authored: Sat Jul 23 12:21:02 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 12:21:02 2016 +0530
----------------------------------------------------------------------
.../carbon/querystatistics/QueryStatistic.java | 12 ++++++++++
.../AbstractDetailQueryResultIterator.java | 25 +++++++++++++-------
.../iterator/DetailQueryResultIterator.java | 8 ++++++-
.../carbondata/spark/rdd/CarbonQueryRDD.scala | 18 +++++++++++---
4 files changed, 51 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
index 41f3d22..80398c1 100644
--- a/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
+++ b/core/src/main/java/org/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -61,6 +61,18 @@ public class QueryStatistic implements Serializable {
}
/**
+ * Below method will be used to add fixed time statistic.
+ * For example total time taken for scan or result preparation
+ *
+ * @param message statistic message
+ * @param timetaken
+ */
+ public void addFixedTimeStatistic(String message, long timetaken) {
+ this.timeTaken = timetaken;
+ this.message = message;
+ }
+
+ /**
* Below method will be used to get the statistic message, which will
* be used to log
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
index 01e94c8..dc94704 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
@@ -66,14 +66,18 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
*/
protected int[] blockIndexToBeExecuted;
/**
- * to store the statistics
- */
- protected QueryStatistic statistic;
- /**
* total number of records processed
*/
protected long totalNumberOfOutputRecords;
/**
+ * total scan time of the query
+ */
+ protected long totalScanTime;
+ /**
+ * total time taken to prepare the result
+ */
+ protected long totalResultPreparationTime;
+ /**
* number of cores which can be used
*/
private long numberOfCores;
@@ -113,7 +117,6 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
executor = queryExecutor;
this.blockExecutionInfos = infos;
this.blockIndexToBeExecuted = new int[(int) numberOfCores];
- statistic = new QueryStatistic();
intialiseInfos();
}
@@ -142,9 +145,15 @@ public abstract class AbstractDetailQueryResultIterator extends CarbonIterator {
if (currentCounter < totalNumberOfNode) {
return true;
}
- statistic.addStatistics("Time taken to processed " + blockExecutionInfos.size()
- + " block(s) of output record size: " + totalNumberOfOutputRecords,
- System.currentTimeMillis());
+ QueryStatistic statistic = new QueryStatistic();
+ statistic
+ .addFixedTimeStatistic("Time taken to scan " + blockExecutionInfos.size() + " block(s) ",
+ totalScanTime);
+ blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
+ statistic = new QueryStatistic();
+ statistic.addFixedTimeStatistic(
+ "Time take to prepare query result of size " + totalNumberOfOutputRecords,
+ totalResultPreparationTime);
blockExecutionInfos.get(0).getStatisticsRecorder().recordStatistics(statistic);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
index db6e9aa..e2f54ee 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
@@ -55,8 +55,11 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
@Override public BatchResult next() {
currentCounter += updateSliceIndexToBeExecuted();
CarbonIterator<Result> result = null;
+ long startTime = System.currentTimeMillis();
+ BatchResult batchResult = null;
try {
result = executor.executeQuery(blockExecutionInfos, blockIndexToBeExecuted);
+ totalScanTime+=System.currentTimeMillis()-startTime;
} catch (QueryExecutionException e) {
throw new RuntimeException(e.getCause().getMessage());
}
@@ -71,7 +74,10 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
Result next = result.next();
if (next.size() > 0) {
totalNumberOfOutputRecords += next.size();
- return queryResultPreparator.prepareQueryResult(next);
+ startTime=System.currentTimeMillis();
+ batchResult = queryResultPreparator.prepareQueryResult(next);
+ totalResultPreparationTime+=System.currentTimeMillis()-startTime;
+ return batchResult;
} else {
return new BatchResult();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a81ed981/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
index e0ec5fe..97f1993 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
@@ -18,7 +18,6 @@
package org.carbondata.spark.rdd
-
import java.util
import scala.collection.JavaConverters._
@@ -44,6 +43,7 @@ import org.carbondata.spark.Value
import org.carbondata.spark.load.CarbonLoaderUtil
import org.carbondata.spark.util.QueryPlanUtil
+
class CarbonSparkPartition(rddId: Int, val idx: Int,
val locations: Array[String],
val tableBlockInfos: util.List[TableBlockInfo])
@@ -216,7 +216,13 @@ class CarbonQueryRDD[V: ClassTag](
}
if (finished) {
clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
+ if (null != queryModel.getStatisticsRecorder) {
+ val queryStatistic = new QueryStatistic
+ queryStatistic
+ .addStatistics("Total Time taken to execute the query in executor Side",
+ System.currentTimeMillis - queryStartTime
+ )
+ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
queryModel.getStatisticsRecorder.logStatistics();
}
}
@@ -231,7 +237,13 @@ class CarbonQueryRDD[V: ClassTag](
recordCount += 1
if (queryModel.getLimit != -1 && recordCount >= queryModel.getLimit) {
clearDictionaryCache(queryModel.getColumnToDictionaryMapping)
- if(null!=queryModel.getStatisticsRecorder) {
+ if (null != queryModel.getStatisticsRecorder) {
+ val queryStatistic = new QueryStatistic
+ queryStatistic
+ .addStatistics("Total Time taken to execute the query in executor Side",
+ System.currentTimeMillis - queryStartTime
+ )
+ queryModel.getStatisticsRecorder.recordStatistics(queryStatistic);
queryModel.getStatisticsRecorder.logStatistics();
}
}
[06/47] incubator-carbondata git commit: [CARBONDATA-94] fixed load
data when csv file record delimiter is '|' and data with header
Posted by ra...@apache.org.
[CARBONDATA-94] fixed load data when csv file record delimiter is '|' and data with header
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4cff504c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4cff504c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4cff504c
Branch: refs/heads/master
Commit: 4cff504cfb538fb7a3ebf3755df2ea968eb79b8b
Parents: 154a4d3
Author: Gin-zhj <zh...@huawei.com>
Authored: Fri Jul 22 21:18:53 2016 +0800
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Fri Jul 22 18:48:53 2016 +0530
----------------------------------------------------------------------
.../spark/src/test/resources/datadelimiter.csv | 11 +++++++
.../dataload/TestLoadDataWithHiveSyntax.scala | 32 +++++++++++++++++++-
.../processing/csvload/GraphExecutionUtil.java | 31 +++++++++++++++++++
3 files changed, 73 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4cff504c/integration/spark/src/test/resources/datadelimiter.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datadelimiter.csv b/integration/spark/src/test/resources/datadelimiter.csv
new file mode 100644
index 0000000..bc06817
--- /dev/null
+++ b/integration/spark/src/test/resources/datadelimiter.csv
@@ -0,0 +1,11 @@
+empno|empname|designation|doj|workgroupcategory|workgroupcategoryname|deptno|deptname|projectcode|projectjoindate|projectenddate|attendance|utilization|salary
+11|arvind|SE|17-01-2007|1|developer|10|network|928478|17-02-2007|29-11-2016|96|96.2|5040.56
+12|krithin|SSE|29-05-2008|1|developer|11|protocol|928378|29-06-2008|30-12-2016|85|95.1|7124.21
+13|madhan|TPL|07-07-2009|2|tester|10|network|928478|07-08-2009|30-12-2016|88|99|9054.235
+14|anandh|SA|29-12-2010|3|manager|11|protocol|928278|29-01-2011|29-06-2016|77|92.2|11248.25
+15|ayushi|SSA|09-11-2011|1|developer|12|security|928375|09-12-2011|29-05-2016|99|91.5|13245.48
+16|pramod|SE|14-10-2012|1|developer|13|configManagement|928478|14-11-2012|29-12-2016|86|93|5040.56
+17|gawrav|PL|22-09-2013|2|tester|12|security|928778|22-10-2013|15-11-2016|78|97.45|9574.24
+18|sibi|TL|15-08-2014|2|tester|14|Learning|928176|15-09-2014|29-05-2016|84|98.23|7245.25
+19|shivani|PL|12-05-2015|1|developer|10|network|928977|12-06-2015|12-11-2016|88|91.678|11254.24
+20|bill|PM|01-12-2015|3|manager|14|Learning|928479|01-01-2016|30-11-2016|75|94.22|13547.25
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4cff504c/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index df75c4b..99bae17 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -47,6 +47,9 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
"projectcode int, projectjoindate String,projectenddate String, attendance String," +
"utilization String,salary String)row format delimited fields terminated by ','"
)
+
+ sql("drop table if exists carbontable1")
+ sql("drop table if exists hivetable1")
}
test("test data loading and validate query output") {
@@ -547,11 +550,38 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select id from t3 where salary = 15000"),Seq(Row(1)))
}
+ test("test data loading when delimiter is '|' and data with header") {
+ sql(
+ "CREATE table carbontable1 (empno int, empname String, designation String, doj String, " +
+ "workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, " +
+ "projectcode int, projectjoindate String, projectenddate String,attendance double," +
+ "utilization double,salary double) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES" +
+ "('DICTIONARY_EXCLUDE'='empno,empname,designation,doj,workgroupcategory," +
+ "workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate')"
+ )
+ sql(
+ "create table hivetable1 (empno int, empname String, designation string, doj String, " +
+ "workgroupcategory int, workgroupcategoryname String,deptno int, deptname String, " +
+ "projectcode int, projectjoindate String,projectenddate String, attendance double," +
+ "utilization double,salary double)row format delimited fields terminated by ','"
+ )
+
+ sql(
+ "LOAD DATA local inpath './src/test/resources/datadelimiter.csv' INTO TABLE carbontable1 OPTIONS" +
+ "('DELIMITER'= '|', 'QUOTECHAR'= '\"')"
+ )
+
+ sql("LOAD DATA local inpath './src/test/resources/datawithoutheader.csv' INTO table hivetable1")
+
+ checkAnswer(sql("select * from carbontable1"), sql("select * from hivetable1"))
+ }
+
override def afterAll {
sql("drop table carbontable")
sql("drop table hivetable")
sql("drop table if exists header_test")
sql("drop table if exists mixed_header_test")
-
+ sql("drop table carbontable1")
+ sql("drop table hivetable1")
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4cff504c/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java
index b0b74c4..27a1f5c 100644
--- a/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java
+++ b/processing/src/main/java/org/carbondata/processing/csvload/GraphExecutionUtil.java
@@ -126,6 +126,7 @@ public final class GraphExecutionUtil {
}
if (null != readLine) {
+ delimiter = delimiterConverter(delimiter);
String[] columnNames = readLine.split(delimiter);
TextFileInputField[] textFileInputFields = new TextFileInputField[columnNames.length];
@@ -289,6 +290,7 @@ public final class GraphExecutionUtil {
String readLine = readCSVFile(csvFilePath);
if (null != readLine) {
+ delimiter = delimiterConverter(delimiter);
String[] columnFromCSV = readLine.toLowerCase().split(delimiter);
List<String> csvColumnsList = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
@@ -325,4 +327,33 @@ public final class GraphExecutionUtil {
}
return columnNames;
}
+
+ /**
+ * special char delimiter Converter
+ *
+ * @param delimiter
+ * @return delimiter
+ */
+ public static String delimiterConverter(String delimiter) {
+ switch (delimiter) {
+ case "|":
+ case "*":
+ case ".":
+ case ":":
+ case "^":
+ case "\\":
+ case"$":
+ case "+":
+ case "?":
+ case "(":
+ case ")":
+ case "{":
+ case "}":
+ case "[":
+ case "]":
+ return "\\" + delimiter;
+ default:
+ return delimiter;
+ }
+ }
}
[37/47] incubator-carbondata git commit: [CARBONDATA-122] Block
distribution give second and third preferred locaiton (#887)
Posted by ra...@apache.org.
[CARBONDATA-122] Block distribution give second and third preferred locaiton (#887)
Give second and third option for node distribution
Support second and third locations option for dataload
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/066f74b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/066f74b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/066f74b6
Branch: refs/heads/master
Commit: 066f74b649257b02e0e0b07c34ddc61661f267b4
Parents: 5341c7d
Author: Venkata Ramana G <g....@gmail.com>
Authored: Fri Jul 29 07:57:02 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Fri Jul 29 07:57:02 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/core/load/BlockDetails.java | 8 ++++-
.../hadoop/test/util/StoreCreator.java | 2 +-
.../org/apache/spark/util/SplitUtils.scala | 6 +++-
.../spark/rdd/CarbonDataLoadRDD.scala | 29 +++++++++++++++--
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../carbondata/spark/rdd/CarbonQueryRDD.scala | 33 +++++++++++++++++---
6 files changed, 69 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/core/src/main/java/org/carbondata/core/load/BlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/load/BlockDetails.java b/core/src/main/java/org/carbondata/core/load/BlockDetails.java
index 9f0e13b..39c39a0 100644
--- a/core/src/main/java/org/carbondata/core/load/BlockDetails.java
+++ b/core/src/main/java/org/carbondata/core/load/BlockDetails.java
@@ -38,11 +38,14 @@ public class BlockDetails implements Serializable {
private long blockLength;
//file path which block belong to
private String filePath;
+ // locations where this block exists
+ private String[] locations;
- public BlockDetails(String filePath, long blockOffset, long blockLength) {
+ public BlockDetails(String filePath, long blockOffset, long blockLength, String[] locations) {
this.filePath = filePath;
this.blockOffset = blockOffset;
this.blockLength = blockLength;
+ this.locations = locations;
}
public long getBlockOffset() {
@@ -69,4 +72,7 @@ public class BlockDetails implements Serializable {
this.filePath = filePath;
}
+ public String[] getLocations() {
+ return locations;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java
index c419c27..15b71ab 100644
--- a/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/carbondata/hadoop/test/util/StoreCreator.java
@@ -365,7 +365,7 @@ public class StoreCreator {
schmaModel.setCsvFilePath(loadModel.getFactFilePath());
SchemaInfo info = new SchemaInfo();
BlockDetails blockDetails = new BlockDetails(loadModel.getFactFilePath(),
- 0, new File(loadModel.getFactFilePath()).length());
+ 0, new File(loadModel.getFactFilePath()).length(), new String[] {"localhost"});
GraphGenerator.blockInfo.put("qwqwq", new BlockDetails[] { blockDetails });
schmaModel.setBlocksID("qwqwq");
schmaModel.setEscapeCharacter("\\");
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/integration/spark/src/main/scala/org/apache/spark/util/SplitUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/util/SplitUtils.scala b/integration/spark/src/main/scala/org/apache/spark/util/SplitUtils.scala
index c126f0e..296ce91 100644
--- a/integration/spark/src/main/scala/org/apache/spark/util/SplitUtils.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/util/SplitUtils.scala
@@ -56,7 +56,11 @@ object SplitUtils {
part.asInstanceOf[NewHadoopPartition].serializableHadoopSplit.value.asInstanceOf[FileSplit]
}
splits.map { block =>
- new BlockDetails(block.getPath.toString, block.getStart, block.getLength)
+ new BlockDetails(block.getPath.toString,
+ block.getStart,
+ block.getLength,
+ block.getLocations
+ )
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 36a1c0f..2a19da9 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -19,6 +19,7 @@
package org.carbondata.spark.rdd
import java.lang.Long
+import java.util
import java.util.UUID
import scala.collection.JavaConverters._
@@ -452,9 +453,31 @@ class CarbonDataLoadRDD[K, V](
case false =>
// for node partition
val theSplit = split.asInstanceOf[CarbonNodePartition]
- val location: Seq[String] = List(theSplit.serializableHadoopSplit)
- logInfo("Prefered Location for split : " + location(0))
- location
+ val firstOptionLocation: Seq[String] = List(theSplit.serializableHadoopSplit)
+ logInfo("Preferred Location for split : " + firstOptionLocation(0))
+ val blockMap = new util.LinkedHashMap[String, Integer]()
+ val tableBlocks = theSplit.blocksDetails
+ tableBlocks.foreach(tableBlock => tableBlock.getLocations.foreach(
+ location => {
+ if (!firstOptionLocation.exists(location.equalsIgnoreCase(_))) {
+ val currentCount = blockMap.get(location)
+ if (currentCount == null) {
+ blockMap.put(location, 1)
+ } else {
+ blockMap.put(location, currentCount + 1)
+ }
+ }
+ }
+ )
+ )
+
+ val sortedList = blockMap.entrySet().asScala.toSeq.sortWith((nodeCount1, nodeCount2) => {
+ nodeCount1.getValue > nodeCount2.getValue
+ }
+ )
+
+ val sortedNodesList = sortedList.map(nodeCount => nodeCount.getKey).take(2)
+ firstOptionLocation ++ sortedNodesList
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 54cf1bd..ab77ea9 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -768,7 +768,7 @@ object CarbonDataRDDFactory extends Logging {
entry._2.asScala.map(distributable => {
val tableBlock = distributable.asInstanceOf[TableBlockInfo]
new BlockDetails(tableBlock.getFilePath,
- tableBlock.getBlockOffset, tableBlock.getBlockLength
+ tableBlock.getBlockOffset, tableBlock.getBlockLength, tableBlock.getLocations
)
}).toArray
(entry._1, blockDetailsList)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/066f74b6/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
index 50d3ca7..9eb680c 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonQueryRDD.scala
@@ -277,8 +277,33 @@ class CarbonQueryRDD[V: ClassTag](
/**
* Get the preferred locations where to launch this task.
*/
- override def getPreferredLocations(partition: Partition): Seq[String] = {
- val theSplit = partition.asInstanceOf[CarbonSparkPartition]
- theSplit.locations.filter(_ != "localhost")
- }
+ override def getPreferredLocations(partition: Partition): Seq[String] = {
+ val theSplit = partition.asInstanceOf[CarbonSparkPartition]
+ val firstOptionLocation = theSplit.locations.filter(_ != "localhost")
+ val tableBlocks = theSplit.tableBlockInfos
+ // node name and count mapping
+ val blockMap = new util.LinkedHashMap[String, Integer]()
+
+ tableBlocks.asScala.foreach(tableBlock => tableBlock.getLocations.foreach(
+ location => {
+ if (!firstOptionLocation.exists(location.equalsIgnoreCase(_))) {
+ val currentCount = blockMap.get(location)
+ if (currentCount == null) {
+ blockMap.put(location, 1)
+ } else {
+ blockMap.put(location, currentCount + 1)
+ }
+ }
+ }
+ )
+ )
+
+ val sortedList = blockMap.entrySet().asScala.toSeq.sortWith((nodeCount1, nodeCount2) => {
+ nodeCount1.getValue > nodeCount2.getValue
+ }
+ )
+
+ val sortedNodesList = sortedList.map(nodeCount => nodeCount.getKey).take(2)
+ firstOptionLocation ++ sortedNodesList
+ }
}
[13/47] incubator-carbondata git commit: [CARBONDATA-99] For complex
type filter queries if query contains filter expression rather than
BinaryExpression the system was not able to get the dimensions which are
involved (#854)
Posted by ra...@apache.org.
[CARBONDATA-99] For complex type filter queries if query contains filter expression rather than BinaryExpression the system was not able to get the dimensions which are involved (#854)
in the particular filter expression for executing complex type filter column expressions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7159dce7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7159dce7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7159dce7
Branch: refs/heads/master
Commit: 7159dce7b17646c67c9eed36f4d22d2577940055
Parents: fb8d807
Author: sujith71955 <su...@gmail.com>
Authored: Sun Jul 24 21:44:46 2016 +0530
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Sun Jul 24 21:44:46 2016 +0530
----------------------------------------------------------------------
.../query/carbon/executor/util/QueryUtil.java | 20 ++++++++++++++++++++
.../filter/resolver/AndFilterResolverImpl.java | 5 +++--
.../resolver/LogicalFilterResolverImpl.java | 11 +++++++----
.../filters/FilterExpressionProcessor.java | 12 +++++-------
integration/spark/src/test/resources/array1.csv | 2 ++
.../complexType/TestComplexTypeQuery.scala | 19 ++++++++++++++++++-
6 files changed, 55 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index 2433cbc..b1de455 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -1108,6 +1108,8 @@ public class QueryUtil {
if (filterExpression instanceof BinaryLogicalExpression) {
BinaryLogicalExpression logicalExpression = (BinaryLogicalExpression) filterExpression;
dimensionResolvedInfos.addAll(logicalExpression.getColumnList());
+ } else {
+ addColumnDimensions(filterExpression, filterDimensions);
}
for (ColumnExpression info : dimensionResolvedInfos) {
if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
@@ -1118,4 +1120,22 @@ public class QueryUtil {
}
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively and add the dimension instance to the set which holds the dimension
+ * instances of the complex filter expressions.
+ *
+ * @param filterDimensions
+ * @return
+ */
+ private static void addColumnDimensions(Expression expression,
+ Set<CarbonDimension> filterDimensions) {
+ if (null != expression && expression instanceof ColumnExpression) {
+ filterDimensions.add(((ColumnExpression) expression).getDimension());
+ return;
+ }
+ for (Expression child : expression.getChildren()) {
+ addColumnDimensions(child, filterDimensions);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java b/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
index 442b17e..62ebf99 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
@@ -24,6 +24,7 @@ import org.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
import org.carbondata.query.carbonfilterinterface.ExpressionType;
+import org.carbondata.query.expression.BinaryExpression;
public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
@@ -33,8 +34,8 @@ public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
private static final long serialVersionUID = -761688076874662001L;
public AndFilterResolverImpl(FilterResolverIntf leftEvalutor, FilterResolverIntf rightEvalutor,
- ExpressionType filterExpressionType) {
- super(leftEvalutor, rightEvalutor, filterExpressionType);
+ ExpressionType filterExpressionType,BinaryExpression expression) {
+ super(leftEvalutor, rightEvalutor, expression);
}
@Override public void getStartKey(SegmentProperties segmentProperties, long[] startKeys,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
index 46b03e4..b0ec737 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
@@ -25,6 +25,7 @@ import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
import org.carbondata.query.carbonfilterinterface.ExpressionType;
import org.carbondata.query.carbonfilterinterface.FilterExecuterType;
+import org.carbondata.query.expression.BinaryExpression;
import org.carbondata.query.expression.Expression;
import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -40,11 +41,14 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
protected ExpressionType filterExpressionType;
+ private BinaryExpression filterExpression;
+
public LogicalFilterResolverImpl(FilterResolverIntf leftEvalutor,
- FilterResolverIntf rightEvalutor, ExpressionType filterExpressionType) {
+ FilterResolverIntf rightEvalutor,BinaryExpression currentExpression) {
this.leftEvalutor = leftEvalutor;
this.rightEvalutor = rightEvalutor;
- this.filterExpressionType = filterExpressionType;
+ this.filterExpressionType = currentExpression.getFilterExpressionType();
+ this.filterExpression=currentExpression;
}
/**
@@ -104,7 +108,6 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
}
@Override public Expression getFilterExpression() {
- // TODO Auto-generated method stub
- return null;
+ return filterExpression;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
index 2d69010..9482efe 100644
--- a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
@@ -41,7 +41,6 @@ import org.carbondata.query.expression.Expression;
import org.carbondata.query.expression.conditional.BinaryConditionalExpression;
import org.carbondata.query.expression.conditional.ConditionalExpression;
import org.carbondata.query.expression.exception.FilterUnsupportedException;
-import org.carbondata.query.expression.logical.BinaryLogicalExpression;
import org.carbondata.query.filter.executer.FilterExecuter;
import org.carbondata.query.filter.resolver.ConditionalFilterResolverImpl;
import org.carbondata.query.filter.resolver.FilterResolverIntf;
@@ -213,7 +212,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
AbsoluteTableIdentifier tableIdentifier, Expression intermediateExpression) {
ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
BinaryExpression currentExpression = null;
- BinaryLogicalExpression logicalExpression = null;
switch (filterExpressionType) {
case OR:
currentExpression = (BinaryExpression) expressionTree;
@@ -221,14 +219,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
currentExpression),
createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
- currentExpression), filterExpressionType);
+ currentExpression),currentExpression);
case AND:
- logicalExpression = (BinaryLogicalExpression) expressionTree;
+ currentExpression = (BinaryExpression) expressionTree;
return new LogicalFilterResolverImpl(
- createFilterResolverTree(logicalExpression.getLeft(), tableIdentifier,
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
currentExpression),
- createFilterResolverTree(logicalExpression.getRight(), tableIdentifier,
- currentExpression), filterExpressionType);
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
+ currentExpression), currentExpression);
case EQUALS:
case IN:
return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, false, expressionTree,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/integration/spark/src/test/resources/array1.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/array1.csv b/integration/spark/src/test/resources/array1.csv
new file mode 100644
index 0000000..c7a0f89
--- /dev/null
+++ b/integration/spark/src/test/resources/array1.csv
@@ -0,0 +1,2 @@
+1,hello$bye,12345$5678,123456$3456,12.0$13.0,2017-07-09 12:00:00$2016-07-09 13:00:00,123456789$987654321
+2,welcome$hi,9876$1234,13456$356,15.0$18.0,2019-07-09 12:00:00$2015-07-09 13:00:00,8888888888$99999999999
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index ab48133..3f6c3d8 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -37,6 +37,7 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
sql("drop table if exists complexcarbontable").show
sql("drop table if exists complexhivetable").show
+ sql("drop table if exists complex_filter").show
sql("drop table if exists structusingstructCarbon").show
sql("drop table if exists structusingstructHive").show
sql("drop table if exists structusingarraycarbon").show
@@ -45,6 +46,8 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexcarbontable OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', 'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
sql("create table complexhivetable(deviceInformationId int, channelsId string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>, proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId double,contractNumber double)row format delimited fields terminated by ',' collection items terminated by '$' map keys terminated by ':'")
sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table complexhivetable");
+ sql("create table complex_filter(test1 int, test2 array<String>,test3 array<bigint>,test4 array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/array1.csv' INTO TABLE complex_filter options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'= 'test1,test2,test3,test4,test5,test6,test7')").show()
sql("create table structusingarraycarbon (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>) STORED BY 'org.apache.carbondata.format'");
sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarraycarbon options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
@@ -57,7 +60,21 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table structusingstructhive")
}
-
+
+
+
+ test("complex filter set1") {
+ checkAnswer(
+ sql("select test3[1] from complex_filter where test4[1] not like'%1%' order by test1"),
+ Seq(Row(5678), Row(1234))
+ )
+ }
+ test("complex filter set2") {
+ checkAnswer(
+ sql("select test2[0] from complex_filter where test3[0] like '%1234%'"),
+ Seq(Row("hello"))
+ )
+ }
test("select * from structusingarraycarbon") {
checkAnswer(sql("select * from structusingarraycarbon"),
sql("select * from structusingarrayhive"))
[12/47] incubator-carbondata git commit: [CARBONDATA-97] Decimal
Precision and scale is not getting applied based on schema metadata. (#843)
Posted by ra...@apache.org.
[CARBONDATA-97] Decimal Precision and scale is not getting applied based on schema metadata. (#843)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fb8d8070
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fb8d8070
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fb8d8070
Branch: refs/heads/master
Commit: fb8d80704068899c3da104c76f5e0a007e7c2305
Parents: c3197a8
Author: sujith71955 <su...@gmail.com>
Authored: Sat Jul 23 14:01:29 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 14:01:29 2016 +0530
----------------------------------------------------------------------
.../apache/spark/sql/CarbonDatasourceRelation.scala | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fb8d8070/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 3035a47..ed4e6cb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -203,14 +203,22 @@ case class CarbonRelation(
cubeMeta.carbonTable.getDimensionByTableName(cubeMeta.carbonTableIdentifier.getTableName)
.asScala.asJava)
sett.asScala.toSeq.filter(!_.getColumnSchema.isInvisible).map(dim => {
- val output: DataType = metaData.carbonTable
- .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName).getDataType
+ val dimval = metaData.carbonTable
+ .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
+ val output: DataType = dimval.getDataType
.toString.toLowerCase match {
case "array" => CarbonMetastoreTypes
.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
case "struct" => CarbonMetastoreTypes
.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
- case dType => CarbonMetastoreTypes.toDataType(dType)
+ case dType =>
+ var dataType = dType
+ if (dimval.getDataType == org.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) {
+ dataType +=
+ "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema
+ .getScale + ")"
+ }
+ CarbonMetastoreTypes.toDataType(dataType)
}
AttributeReference(
[14/47] incubator-carbondata git commit: [Bug] reset statistics
information after dataloading #842
Posted by ra...@apache.org.
[Bug] reset statistics information after dataloading #842
[Bug] reset statistics information after dataloading #842
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1b16f765
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1b16f765
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1b16f765
Branch: refs/heads/master
Commit: 1b16f765c37202284e2b404c5bc45a06c15c23a7
Parents: 7159dce
Author: Zhangshunyu <zh...@huawei.com>
Authored: Mon Jul 25 11:21:15 2016 +0800
Committer: david <qi...@qq.com>
Committed: Mon Jul 25 11:21:15 2016 +0800
----------------------------------------------------------------------
.../core/util/CarbonLoadStatisticsDummy.java | 12 +---
.../core/util/CarbonLoadStatisticsImpl.java | 65 ++++++++------------
.../carbondata/core/util/LoadStatistics.java | 6 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 5 +-
4 files changed, 30 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1b16f765/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsDummy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsDummy.java b/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsDummy.java
index 1561efa..bb82fcd 100644
--- a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsDummy.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsDummy.java
@@ -35,7 +35,7 @@ public class CarbonLoadStatisticsDummy implements LoadStatistics {
}
@Override
- public void recordGlobalDicGenTotalTime(Long glblDicTimePoint) {
+ public void recordDicShuffleAndWriteTime() {
}
@@ -45,16 +45,6 @@ public class CarbonLoadStatisticsDummy implements LoadStatistics {
}
@Override
- public void recordCsvlDicShuffleMaxTime(Long csvlDicShuffleTimePart) {
-
- }
-
- @Override
- public void recordDicWriteFileMaxTime(Long dicWriteFileTimePart) {
-
- }
-
- @Override
public void recordDictionaryValuesTotalTime(String partitionID,
Long dictionaryValuesTotalTimeTimePoint) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1b16f765/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java b/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
index e55179e..3a56db2 100644
--- a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
@@ -50,13 +50,6 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
private long dicShuffleAndWriteFileTotalStartTime = 0;
private long dicShuffleAndWriteFileTotalCostTime = 0;
- //Due to thread thread blocking in each task, we only record the max
- //csvlDicShuffle Time of each single thread
- private long csvlDicShuffleCostTime = 0;
- //Due to thread thread blocking in each task, we only record the max
- //dicWriteFile Time of each single thread
- private long dicWriteFileCostTime = 0;
-
//LRU cache load one time
private double lruCacheLoadTime = 0;
@@ -90,7 +83,7 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
private double totalTime = 0;
@Override
- public void initPartitonInfo(String PartitionId) {
+ public void initPartitonInfo(String PartitionId) {
parDictionaryValuesTotalTimeMap.put(PartitionId, new Long[2]);
parCsvInputStepTimeMap.put(PartitionId, new Long[2]);
parSortRowsStepTotalTimeMap.put(PartitionId, new Long[2]);
@@ -100,13 +93,15 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
}
//Record the time
- public void recordGlobalDicGenTotalTime(Long glblDicTimePoint) {
+ public void recordDicShuffleAndWriteTime() {
+ Long dicShuffleAndWriteTimePoint = System.currentTimeMillis();
if (0 == dicShuffleAndWriteFileTotalStartTime) {
- dicShuffleAndWriteFileTotalStartTime = glblDicTimePoint;
+ dicShuffleAndWriteFileTotalStartTime = dicShuffleAndWriteTimePoint;
}
- if (glblDicTimePoint - dicShuffleAndWriteFileTotalStartTime >
+ if (dicShuffleAndWriteTimePoint - dicShuffleAndWriteFileTotalStartTime >
dicShuffleAndWriteFileTotalCostTime) {
- dicShuffleAndWriteFileTotalCostTime = glblDicTimePoint - dicShuffleAndWriteFileTotalStartTime;
+ dicShuffleAndWriteFileTotalCostTime =
+ dicShuffleAndWriteTimePoint - dicShuffleAndWriteFileTotalStartTime;
}
}
@@ -120,19 +115,6 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
}
}
- public void recordCsvlDicShuffleMaxTime(Long csvlDicShuffleTimePart) {
- if (csvlDicShuffleTimePart > csvlDicShuffleCostTime) {
- csvlDicShuffleCostTime = csvlDicShuffleTimePart;
- }
- }
-
- public void recordDicWriteFileMaxTime(Long dicWriteFileTimePart) {
- if (dicWriteFileTimePart > dicWriteFileCostTime) {
- dicWriteFileCostTime = dicWriteFileTimePart;
- }
- }
-
-
public double getLruCacheLoadTime() {
return lruCacheLoadTime;
}
@@ -260,14 +242,6 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
return loadCsvfilesToDfCostTime / 1000.0;
}
- private double getCsvlDicShuffleMaxTime() {
- return csvlDicShuffleCostTime / 1000.0;
- }
-
- private double getDicWriteFileMaxTime() {
- return dicWriteFileCostTime / 1000.0;
- }
-
private double getDictionaryValuesTotalTime(String partitionID) {
return parDictionaryValuesTotalTimeMap.get(partitionID)[1] / 1000.0;
}
@@ -342,12 +316,6 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
double dicShuffleAndWriteFileTotalTime = getDicShuffleAndWriteFileTotalTime();
LOGGER.audit("STAGE 2 ->Global dict shuffle and write dict file: " +
+ dicShuffleAndWriteFileTotalTime + "(s)");
- double csvShuffleMaxTime = getCsvlDicShuffleMaxTime();
- LOGGER.audit("STAGE 2.1 -> |_maximum distinct column shuffle time: "
- + csvShuffleMaxTime + "(s)");
- double dicWriteFileMaxTime = getDicWriteFileMaxTime();
- LOGGER.audit("STAGE 2.2 -> |_maximum distinct column write dict file time: "
- + dicWriteFileMaxTime + "(s)");
}
private void printLruCacheLoadTimeInfo() {
@@ -420,7 +388,26 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
printLoadSpeedInfo(partitionID);
} catch (Exception e) {
LOGGER.audit("Can't print Statistics Information");
+ } finally {
+ resetLoadStatistics();
}
}
+ //Reset the load statistics values
+ private void resetLoadStatistics() {
+ loadCsvfilesToDfStartTime = 0;
+ loadCsvfilesToDfCostTime = 0;
+ dicShuffleAndWriteFileTotalStartTime = 0;
+ dicShuffleAndWriteFileTotalCostTime = 0;
+ lruCacheLoadTime = 0;
+ totalRecords = 0;
+ totalTime = 0;
+ parDictionaryValuesTotalTimeMap.clear();
+ parCsvInputStepTimeMap.clear();
+ parSortRowsStepTotalTimeMap.clear();
+ parGeneratingDictionaryValuesTimeMap.clear();
+ parMdkGenerateTotalTimeMap.clear();
+ parDictionaryValue2MdkAdd2FileTime.clear();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1b16f765/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/LoadStatistics.java b/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
index c353f0c..e5f24e6 100644
--- a/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
+++ b/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
@@ -24,14 +24,10 @@ public interface LoadStatistics {
void initPartitonInfo(String PartitionId);
//Record the time
- void recordGlobalDicGenTotalTime(Long glblDicTimePoint);
+ void recordDicShuffleAndWriteTime();
void recordLoadCsvfilesToDfTime();
- void recordCsvlDicShuffleMaxTime(Long csvlDicShuffleTimePart);
-
- void recordDicWriteFileMaxTime(Long dicWriteFileTimePart);
-
void recordDictionaryValuesTotalTime(String partitionID,
Long dictionaryValuesTotalTimeTimePoint);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1b16f765/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index e519030..7067fd3 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -308,9 +308,7 @@ class CarbonGlobalDictionaryGenerateRDD(
val valuesBuffer = new mutable.HashSet[String]
val rddIter = firstParent[(Int, ColumnDistinctValues)].iterator(split, context)
var rowCount = 0L
- val dicShuffleStartTime = System.currentTimeMillis()
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordGlobalDicGenTotalTime(
- dicShuffleStartTime)
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDicShuffleAndWriteTime()
breakable {
while (rddIter.hasNext) {
val distinctValueList = rddIter.next()._2
@@ -372,6 +370,7 @@ class CarbonGlobalDictionaryGenerateRDD(
sortIndexWriteTask.execute()
}
val sortIndexWriteTime = (System.currentTimeMillis() - t4)
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDicShuffleAndWriteTime()
// After sortIndex writing, update dictionaryMeta
dictWriteTask.updateMetaData()
// clear the value buffer after writing dictionary data
[22/47] incubator-carbondata git commit: Fix some code style (#861)
Posted by ra...@apache.org.
Fix some code style (#861)
Fix code styles
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ad1c9859
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ad1c9859
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ad1c9859
Branch: refs/heads/master
Commit: ad1c9859658cf9941bc444eacee0f8fe66cfff67
Parents: 4b6314c
Author: Zhangshunyu <zh...@huawei.com>
Authored: Mon Jul 25 18:34:47 2016 +0800
Committer: sujith71955 <su...@gmail.com>
Committed: Mon Jul 25 16:04:47 2016 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonSqlParser.scala | 1 -
.../execution/command/carbonTableSchema.scala | 7 ++-----
.../spark/rdd/CarbonDataLoadRDD.scala | 1 -
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 21 ++++++--------------
4 files changed, 8 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ad1c9859/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 79471a9..e222bec 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql
-import java.nio.charset.Charset
import java.util
import java.util.regex.{Matcher, Pattern}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ad1c9859/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 30c022e..5da01ac 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command
import java.io.File
import java.text.SimpleDateFormat
import java.util
-import java.util.{Date, UUID}
+import java.util.UUID
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
@@ -29,10 +29,7 @@ import scala.util.Random
import org.apache.spark.SparkEnv
import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan}
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.types.TimestampType
@@ -59,7 +56,7 @@ import org.carbondata.spark.exception.MalformedCarbonCommandException
import org.carbondata.spark.load._
import org.carbondata.spark.partition.api.impl.QueryPartitionHelper
import org.carbondata.spark.rdd.CarbonDataRDDFactory
-import org.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, GlobalDictionaryUtil}
+import org.carbondata.spark.util.{CarbonScalaUtil, GlobalDictionaryUtil}
import org.carbondata.spark.CarbonSparkFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ad1c9859/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 60a3a8a..87f7885 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -180,7 +180,6 @@ class CarbonDataLoadRDD[K, V](
}
if (storeLocation == null) {
storeLocation = System.getProperty("java.io.tmpdir")
- // storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
}
storeLocation = storeLocation + '/' + System.nanoTime() + '/' + theSplit.index
dataloadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ad1c9859/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
index e2dc900..0129a2c 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -188,7 +188,7 @@ class CarbonMergerRDD[K, V](
var noOfBlocks = 0
- var taskInfoList = new util.ArrayList[Distributable]
+ val taskInfoList = new util.ArrayList[Distributable]
// for each valid segment.
for (eachSeg <- carbonMergerMapping.validSegments) {
@@ -227,15 +227,13 @@ class CarbonMergerRDD[K, V](
)
noOfBlocks += blocksOfOneSegment.size
- var index = 0
- taskIdMapping.asScala.foreach(
+ taskIdMapping.asScala.foreach(
entry =>
taskInfoList.add(new TableTaskInfo(entry._1, entry._2).asInstanceOf[Distributable])
)
}
// send complete list of blocks to the mapping util.
- nodeMapping =
- CarbonLoaderUtil.nodeBlockMapping(taskInfoList, -1)
+ nodeMapping = CarbonLoaderUtil.nodeBlockMapping(taskInfoList, -1)
val confExecutors = confExecutorsTemp.toInt
val requiredExecutors = if (nodeMapping.size > confExecutors) {
@@ -273,13 +271,7 @@ class CarbonMergerRDD[K, V](
.add(new NodeInfo(blocksPerNode.getTaskId, blocksPerNode.getTableBlockInfoList.size))
})
if (list.size() != 0) {
- result
- .add(new CarbonSparkPartition(id,
- i,
- Seq(entry._1).toArray,
- list
- )
- )
+ result.add(new CarbonSparkPartition(id, i, Seq(entry._1).toArray, list))
i += 1
}
}
@@ -287,14 +279,13 @@ class CarbonMergerRDD[K, V](
// print the node info along with task and number of blocks for the task.
nodeTaskBlocksMap.asScala.foreach((entry : (String, List[NodeInfo])) => {
- logInfo(s"for the node $entry._1" )
+ logInfo(s"for the node ${entry._1}" )
for (elem <- entry._2.asScala) {
logInfo("Task ID is " + elem.TaskId + "no. of blocks is " + elem.noOfBlocks)
}
} )
- // val noOfBlocks = blockList.size
- val noOfNodes = nodes.size
+ val noOfNodes = nodes.length
val noOfTasks = result.size
logInfo(s"Identified no.of.Blocks: $noOfBlocks,"
+ s"parallelism: $defaultParallelism , no.of.nodes: $noOfNodes, no.of.tasks: $noOfTasks"
[17/47] incubator-carbondata git commit: no key columns (#856)
Posted by ra...@apache.org.
no key columns (#856)
when create table without key columns, it should throw exception
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f57a95c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f57a95c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f57a95c6
Branch: refs/heads/master
Commit: f57a95c65ddf70e2e5ccda45c16f5aab300facf0
Parents: 9089324
Author: Jay <li...@huawei.com>
Authored: Mon Jul 25 16:13:56 2016 +0800
Committer: sujith71955 <su...@gmail.com>
Committed: Mon Jul 25 13:43:56 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 5 +++++
.../scala/org/apache/spark/sql/CarbonSqlParser.scala | 7 +++++++
.../testsuite/createtable/TestCreateTableSyntax.scala | 13 +++++++++++++
3 files changed, 25 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f57a95c6/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index a7a8c07..739462e 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -736,6 +736,11 @@ public final class CarbonCommonConstants {
public static final String CARBON_MERGE_SORT_PREFETCH = "carbon.merge.sort.prefetch";
public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
+ /**
+ * default name of data base
+ */
+ public static final String DATABASE_DEFAULT_NAME = "default";
+
// tblproperties
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f57a95c6/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 7033b85..4fd5f14 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -498,6 +498,13 @@ class CarbonSqlParser()
val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
fields, tableProperties)
+ if (dims.length == 0) {
+ throw new MalformedCarbonCommandException(s"Table ${dbName.getOrElse(
+ CarbonCommonConstants.DATABASE_DEFAULT_NAME)}.$tableName"
+ + " can not be created without key columns. Please use DICTIONARY_INCLUDE or " +
+ "DICTIONARY_EXCLUDE to set at least one key column " +
+ "if all specified columns are numeric types")
+ }
val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
// column properties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f57a95c6/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
index fe44336..3868928 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
@@ -99,6 +99,19 @@ class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists hivetable")
}
+ test("create carbon table without dimensions") {
+ try {
+ sql("create table carbontable(msr1 int, msr2 double, msr3 bigint, msr4 decimal)" +
+ " stored by 'org.apache.carbondata.format'")
+ assert(false)
+ } catch {
+ case e : MalformedCarbonCommandException => {
+ assert(e.getMessage.equals("Table default.carbontable can not be created without " +
+ "key columns. Please use DICTIONARY_INCLUDE or DICTIONARY_EXCLUDE to " +
+ "set at least one key column if all specified columns are numeric types"))
+ }
+ }
+ }
override def afterAll {
sql("drop table if exists carbontable")
}
[26/47] incubator-carbondata git commit: [Bug] clean redundancy code
of dictionary generation(#871)
Posted by ra...@apache.org.
[Bug] clean redundancy code of dictionary generation(#871)
[Bug] clean redundancy code of dictionary generation(#871)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/14a46b22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/14a46b22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/14a46b22
Branch: refs/heads/master
Commit: 14a46b22dc397f9960fcef7b3fec6a9780f637a5
Parents: e367e12
Author: Zhangshunyu <zh...@huawei.com>
Authored: Thu Jul 28 14:19:34 2016 +0800
Committer: david <qi...@qq.com>
Committed: Thu Jul 28 14:19:34 2016 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonSqlParser.scala | 8 ++-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 3 --
.../spark/tasks/DictionaryWriterTask.scala | 54 ++++++--------------
.../spark/util/GlobalDictionaryUtil.scala | 5 +-
4 files changed, 19 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14a46b22/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index fdfc683..0fd841d 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -354,11 +354,9 @@ class CarbonSqlParser()
case list@Token("TOK_TABCOLLIST", _) =>
val cols = BaseSemanticAnalyzer.getColumns(list, true)
if (cols != null) {
- val dupColsGrp = cols.asScala
- .groupBy(x => x.getName) filter { case (_, colList) => colList
- .size >
- 1
- }
+ val dupColsGrp = cols.asScala.groupBy(x => x.getName) filter {
+ case (_, colList) => colList.size > 1
+ }
if (dupColsGrp.size > 0) {
var columnName: String = ""
dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14a46b22/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 9a94c5b..e0e8cbf 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -33,15 +33,12 @@ import org.apache.spark.sql.Row
import org.carbondata.common.factory.CarbonCommonFactory
import org.carbondata.common.logging.LogServiceFactory
import org.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
-import org.carbondata.core.carbon.metadata.datatype.DataType
-import org.carbondata.core.carbon.metadata.encoder.Encoding
import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
import org.carbondata.core.locks.CarbonLockFactory
import org.carbondata.core.locks.LockUsage
import org.carbondata.core.util.CarbonTimeStatisticsFactory
-import org.carbondata.processing.etl.DataLoadingException
import org.carbondata.spark.load.{CarbonLoaderUtil, CarbonLoadModel}
import org.carbondata.spark.partition.reader.{CSVParser, CSVReader}
import org.carbondata.spark.tasks.DictionaryWriterTask
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14a46b22/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala b/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
index 380e76b..b62558d 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/tasks/DictionaryWriterTask.scala
@@ -61,54 +61,30 @@ class DictionaryWriterTask(valuesBuffer: mutable.HashSet[String],
}
if (values.length >= 1) {
- var preValue = values(0)
if (model.dictFileExists(columnIndex)) {
- if (dictionary.getSurrogateKey(values(0)) == CarbonCommonConstants
- .INVALID_SURROGATE_KEY) {
- val parsedValue = org.carbondata.core.util.DataTypeUtil
- .normalizeColumnValueForItsDataType(values(0),
- model.primDimensions(columnIndex))
- if (null != parsedValue) {
- writer.write(parsedValue)
- distinctValues.add(parsedValue)
- }
- }
- for (i <- 1 until values.length) {
- if (preValue != values(i)) {
- if (dictionary.getSurrogateKey(values(i)) ==
- CarbonCommonConstants.INVALID_SURROGATE_KEY) {
- val parsedValue = org.carbondata.core.util.DataTypeUtil
- .normalizeColumnValueForItsDataType(values(i),
- model.primDimensions(columnIndex))
- if (null != parsedValue) {
- writer.write(parsedValue)
- distinctValues.add(parsedValue)
- preValue = values(i)
- }
- }
- }
- }
-
- } else {
- val parsedValue = org.carbondata.core.util.DataTypeUtil
- .normalizeColumnValueForItsDataType(values(0),
- model.primDimensions(columnIndex))
- if (null != parsedValue) {
- writer.write(parsedValue)
- distinctValues.add(parsedValue)
- }
- for (i <- 1 until values.length) {
- if (preValue != values(i)) {
+ for (value <- values) {
+ if (dictionary.getSurrogateKey(value) ==
+ CarbonCommonConstants.INVALID_SURROGATE_KEY) {
val parsedValue = org.carbondata.core.util.DataTypeUtil
- .normalizeColumnValueForItsDataType(values(i),
+ .normalizeColumnValueForItsDataType(value,
model.primDimensions(columnIndex))
if (null != parsedValue) {
writer.write(parsedValue)
distinctValues.add(parsedValue)
- preValue = values(i)
}
}
}
+
+ } else {
+ for (value <- values) {
+ val parsedValue = org.carbondata.core.util.DataTypeUtil
+ .normalizeColumnValueForItsDataType(value,
+ model.primDimensions(columnIndex))
+ if (null != parsedValue) {
+ writer.write(parsedValue)
+ distinctValues.add(parsedValue)
+ }
+ }
}
}
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14a46b22/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
index 8ad1204..18e777d 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -22,7 +22,6 @@ import java.nio.charset.Charset
import java.util.regex.Pattern
import scala.collection.JavaConverters._
-import scala.collection.mutable
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
import scala.language.implicitConversions
import scala.util.control.Breaks.{break, breakable}
@@ -48,12 +47,11 @@ import org.carbondata.core.datastorage.store.impl.FileFactory
import org.carbondata.core.reader.CarbonDictionaryReader
import org.carbondata.core.util.CarbonProperties
import org.carbondata.core.writer.CarbonDictionaryWriter
-import org.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
import org.carbondata.processing.etl.DataLoadingException
import org.carbondata.spark.load.CarbonLoaderUtil
import org.carbondata.spark.load.CarbonLoadModel
import org.carbondata.spark.partition.reader.CSVWriter
-import org.carbondata.spark.rdd.{ArrayParser, CarbonAllDictionaryCombineRDD, CarbonBlockDistinctValuesCombineRDD, CarbonColumnDictGenerateRDD, CarbonDataRDDFactory, CarbonGlobalDictionaryGenerateRDD, ColumnPartitioner, DataFormat, DictionaryLoadModel, GenericParser, PrimitiveParser, StructParser}
+import org.carbondata.spark.rdd._
import org.carbondata.spark.CarbonSparkFactory
/**
@@ -100,7 +98,6 @@ object GlobalDictionaryUtil extends Logging {
encoding: Encoding,
excludeEncoding: Encoding): Boolean = {
if (dimension.isComplex()) {
- var has = false
val children = dimension.getListOfChildDimensions
children.asScala.exists(hasEncoding(_, encoding, excludeEncoding))
} else {
[16/47] incubator-carbondata git commit: clean scala code style (#703)
Posted by ra...@apache.org.
clean scala code style (#703)
clean scala code style (#703)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/90893247
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/90893247
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/90893247
Branch: refs/heads/master
Commit: 90893247f3a07df2d5b5435014445f3945ff63e4
Parents: 9f269d1
Author: Gin-zhj <zh...@huawei.com>
Authored: Mon Jul 25 11:59:31 2016 +0800
Committer: david <qi...@qq.com>
Committed: Mon Jul 25 11:59:31 2016 +0800
----------------------------------------------------------------------
.../spark/sql/common/util/CsvCompare.scala | 1 -
.../org/apache/spark/sql/CarbonSqlParser.scala | 2 +-
.../spark/sql/SparkUnknownExpression.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 20 +++---
.../spark/rdd/CarbonDataLoadRDD.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../apache/spark/sql/TestCarbonSqlParser.scala | 72 ++++++++++----------
.../DataCompactionCardinalityBoundryTest.scala | 6 +-
.../datacompaction/DataCompactionTest.scala | 6 +-
.../dataload/TestLoadDataWithHiveSyntax.scala | 4 +-
...estampDataTypeDirectDictionaryTestCase.scala | 3 +-
...TypeDirectDictionaryWithNoDictTestCase.scala | 3 +-
.../TimestampDataTypeNullDataTest.scala | 2 +-
.../util/GlobalDictionaryUtilTestCase.scala | 4 +-
14 files changed, 64 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration-testcases/src/test/scala/org/apache/spark/sql/common/util/CsvCompare.scala
----------------------------------------------------------------------
diff --git a/integration-testcases/src/test/scala/org/apache/spark/sql/common/util/CsvCompare.scala b/integration-testcases/src/test/scala/org/apache/spark/sql/common/util/CsvCompare.scala
index 6035c84..7d8759d 100644
--- a/integration-testcases/src/test/scala/org/apache/spark/sql/common/util/CsvCompare.scala
+++ b/integration-testcases/src/test/scala/org/apache/spark/sql/common/util/CsvCompare.scala
@@ -24,7 +24,6 @@ class CsvCompare {
val carbon = new HashMap[String, Integer]()
val hivefile = new BufferedReader(new FileReader(hive))
val olapfile = new BufferedReader(new FileReader(olap))
- var line: String = null
for(line <- Source.fromFile(file1).getLines())
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 9bc2bb9..7033b85 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -496,7 +496,7 @@ class CarbonSqlParser()
tableProperties: Map[String, String]): tableModel
= {
- var (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
+ val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
fields, tableProperties)
val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index 35e1035..058cea5 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -90,7 +90,7 @@ class SparkUnknownExpression(sparkExp: SparkExpression)
}
def isSingleDimension(): Boolean = {
- var lst = new java.util.ArrayList[ColumnExpression]()
+ val lst = new java.util.ArrayList[ColumnExpression]()
getAllColumnListFromExpressionTree(sparkExp, lst)
if (lst.size == 1 && lst.get(0).isDimension) {
true
@@ -127,7 +127,7 @@ class SparkUnknownExpression(sparkExp: SparkExpression)
}
def isDirectDictionaryColumns(): Boolean = {
- var lst = new ArrayList[ColumnExpression]()
+ val lst = new ArrayList[ColumnExpression]()
getAllColumnListFromExpressionTree(sparkExp, lst)
if (lst.get(0).getCarbonColumn.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
true
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 3a0e21c..0278a68 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -204,7 +204,7 @@ class TableNewProcessor(cm: tableModel, sqlContext: SQLContext) {
if (dataType == DataType.TIMESTAMP) {
encoders.add(Encoding.DIRECT_DICTIONARY)
}
- var colPropMap = new java.util.HashMap[String, String]()
+ val colPropMap = new java.util.HashMap[String, String]()
if (None != cm.colProps && null != cm.colProps.get.get(colName)) {
val colProps = cm.colProps.get.get(colName)
colProps.asScala.foreach { x => colPropMap.put(x.key, x.value) }
@@ -1197,7 +1197,7 @@ private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) e
def run(sqlContext: SQLContext): Seq[Row] = {
// TODO : Implement it.
- var tableName = alterTableModel.tableName
+ val tableName = alterTableModel.tableName
val schemaName = getDB.getDatabaseName(alterTableModel.dbName, sqlContext)
if (null == org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
.getCarbonTable(schemaName + "_" + tableName)) {
@@ -1354,7 +1354,7 @@ private[sql] case class DeleteLoadsById(
}
val path = carbonTable.getMetaDataFilepath
- var segmentStatusManager =
+ val segmentStatusManager =
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
val invalidLoadIds = segmentStatusManager.updateDeletionStatus(loadids.asJava, path).asScala
@@ -1413,9 +1413,9 @@ private[sql] case class DeleteLoadsByLoadDate(
throw new MalformedCarbonCommandException(errorMessage)
}
- var carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
+ val carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
.getCarbonTable(schemaName + '_' + tableName)
- var segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
+ val segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
if (null == carbonTable) {
var relation = CarbonEnv.getInstance(sqlContext).carbonCatalog.lookupRelation1(
@@ -1424,9 +1424,9 @@ private[sql] case class DeleteLoadsByLoadDate(
None
)(sqlContext).asInstanceOf[CarbonRelation]
}
- var path = carbonTable.getMetaDataFilepath()
+ val path = carbonTable.getMetaDataFilepath()
- var invalidLoadTimestamps = segmentStatusManager
+ val invalidLoadTimestamps = segmentStatusManager
.updateDeletionStatus(loadDate, path, timeObj.asInstanceOf[java.lang.Long]).asScala
if(invalidLoadTimestamps.isEmpty) {
LOGGER.audit(s"Delete load by load date is successfull for $schemaName.$tableName.")
@@ -1505,7 +1505,7 @@ private[sql] case class LoadCube(
// Need to fill dimension relation
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
var storeLocation = ""
- var configuredStore = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
+ val configuredStore = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
if (null != configuredStore && configuredStore.length > 0) {
storeLocation = configuredStore(Random.nextInt(configuredStore.length))
}
@@ -1945,7 +1945,7 @@ private[sql] case class ShowLoads(
}
val path = carbonTable.getMetaDataFilepath()
- var segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
+ val segmentStatusManager = new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier)
val loadMetadataDetailsArray = segmentStatusManager.readLoadMetadata(path)
@@ -2110,7 +2110,7 @@ private[sql] case class DeleteLoadByDate(
val relation = CarbonEnv.getInstance(sqlContext).carbonCatalog
.lookupRelation1(Some(schemaName), cubeName, None)(sqlContext).asInstanceOf[CarbonRelation]
var level: String = ""
- var carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata
+ val carbonTable = org.carbondata.core.carbon.metadata.CarbonMetadata
.getInstance().getCarbonTable(schemaName + '_' + cubeName)
if (relation == null) {
LOGGER.audit(s"The delete load by date is failed. Table $schemaName.$cubeName does not exist")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 18ea3b0..8869368 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -174,7 +174,7 @@ class CarbonDataLoadRDD[K, V](
CarbonProperties.getInstance().addProperty("high.cardinality.value", "100000")
CarbonProperties.getInstance().addProperty("is.compressed.keyblock", "false")
CarbonProperties.getInstance().addProperty("carbon.leaf.node.size", "120000")
- var storeLocations = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
+ val storeLocations = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
if (null != storeLocations && storeLocations.length > 0) {
storeLocation = storeLocations(Random.nextInt(storeLocations.length))
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index b314f88..7726739 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -533,7 +533,7 @@ object CarbonDataRDDFactory extends Logging {
)
var storeLocation = ""
- var configuredStore = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
+ val configuredStore = CarbonLoaderUtil.getConfiguredLocalDirs(SparkEnv.get.conf)
if (null != configuredStore && configuredStore.length > 0) {
storeLocation = configuredStore(Random.nextInt(configuredStore.length))
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index 983d488..23c4a9d 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -71,7 +71,7 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-updateColumnGroupsInField") {
val colGroupStr = "(col2,col3),(col5,col6),(col7,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
@@ -93,7 +93,7 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-ColumnGroupsInvalidField_Shouldnotallow") {
val colGroupStr = "(col1,col2),(col10,col6),(col7,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
try {
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -106,7 +106,7 @@ class TestCarbonSqlParser extends QueryTest {
//col1 is measure
val colGroupStr = "(col1,col2),(col5,col6),(col7,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
try {
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -120,7 +120,7 @@ class TestCarbonSqlParser extends QueryTest {
val colGroupStr = "(col2,col3),(col5,col6),(col7,col8)"
val noDictStr = "col5"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr, CarbonCommonConstants.DICTIONARY_EXCLUDE -> noDictStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
try {
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -132,7 +132,7 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-SameColumnInDifferentGroup_ShouldNotAllow") {
val colGroupStr = "(col2,col3),(col5,col6),(col6,col7,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
try {
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -145,7 +145,7 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-ColumnAreNotTogetherAsInSchema_ShouldNotAllow") {
val colGroupStr = "(col2,col3),(col5,col8)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
try {
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -157,7 +157,7 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-ColumnInColumnGroupAreShuffledButInSequence") {
val colGroupStr = "(col2,col3),(col7,col8,col6)"
val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
@@ -167,7 +167,7 @@ class TestCarbonSqlParser extends QueryTest {
// Testing the column group Splitting method with empty table properties so null will be returned.
test("Test-Empty-updateColumnGroupsInField") {
val tableProperties = Map("" -> "")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
val colgrps = stub.updateColumnGroupsInFieldTest(fields, Map())
//assert( rtn === 1)
@@ -177,10 +177,10 @@ class TestCarbonSqlParser extends QueryTest {
// Testing the extracting of Dims and no Dictionary
test("Test-extractDimColsAndNoDictionaryFields") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
+ val (dimCols, noDictionary) = stub.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
// testing col
@@ -197,11 +197,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields1") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -220,11 +220,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields2") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -242,11 +242,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields3") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -265,11 +265,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields4") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col3", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 6)
@@ -288,11 +288,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields5") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col4", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -311,11 +311,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields6") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -336,11 +336,11 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2 ,col1 ",
CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3 ,col4 "
)
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -360,11 +360,11 @@ class TestCarbonSqlParser extends QueryTest {
test("Test-DimAndMsrColsWithNoDictionaryFields8") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE-> "col2,col4", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var (dimCols, noDictionary) = stub
+ val (dimCols, noDictionary) = stub
.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -385,9 +385,9 @@ class TestCarbonSqlParser extends QueryTest {
// Testing the extracting of measures
test("Test-extractMsrColsFromFields") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
- var fields: Seq[Field] = loadAllFields
+ val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- var msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
// testing col
assert(msrCols.lift(0).get.column.equalsIgnoreCase("col1"))
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
index f5fc8e9..7eeaffb 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
@@ -33,12 +33,12 @@ class DataCompactionCardinalityBoundryTest extends QueryTest with BeforeAndAfter
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+ val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
// loading the rows greater than 256. so that the column cardinality crosses byte boundary.
- var csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compactioncard2.csv"
+ val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compactioncard2.csv"
- var csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
+ val csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE cardinalityTest OPTIONS" +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
index 9f87ada..1320b39 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
@@ -34,10 +34,10 @@ class DataCompactionTest extends QueryTest with BeforeAndAfterAll {
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+ val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
- var csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv"
- var csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
+ val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv"
+ val csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE normalcompaction OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
index 99bae17..173fe6b 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntax.scala
@@ -216,7 +216,7 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
)
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/data_withCAPSHeader.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/data_withCAPSHeader.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO table header_test OPTIONS " +
"('DELIMITER'=',', 'QUOTECHAR'='\"')");
checkAnswer(sql("select empno from header_test"),
@@ -248,7 +248,7 @@ class TestLoadDataWithHiveSyntax extends QueryTest with BeforeAndAfterAll {
)
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/data_withMixedHeader.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/data_withMixedHeader.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO table mixed_header_test OPTIONS " +
"('DELIMITER'=',', 'QUOTECHAR'='\"')");
checkAnswer(sql("select empno from mixed_header_test"),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index 606e04b..baff810 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -61,10 +61,9 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryCube OPTIONS" +
"('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
-
} catch {
case x: Throwable => CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index dafe8ee..a62c632 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -61,10 +61,9 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS"
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
-
} catch {
case x: Throwable => CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
index c8c1f81..5260fe7 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -59,7 +59,7 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
.getCanonicalPath
- var csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv"
+ val csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv"
sql("LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE timestampTyeNullData").collect();
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/90893247/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
index f1a076b..4d68e98 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
@@ -153,7 +153,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
test("[issue-80]Global Dictionary Generation") {
- var carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null, null)
+ val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null, null)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
@@ -171,7 +171,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
test("[Issue-190]load csv file without header And support complex type") {
val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
"proddate,gamePointId,contractNumber"
- var carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, null, header)
+ val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, null, header)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
[30/47] incubator-carbondata git commit: [CARBONDATA-116]ignore the
major compacted segment for minor compaction (#880)
Posted by ra...@apache.org.
[CARBONDATA-116]ignore the major compacted segment for minor compaction (#880)
Once a segment is formed using major compaction the same segment can not be considered for minor compaction. this PR will skip the same for an effective compaction.
* for correcting test case.
* for test case correction.
* correcting test cases.
* correcting test cases.
* correcting test cases.
* fixing review comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bdc1321d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bdc1321d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bdc1321d
Branch: refs/heads/master
Commit: bdc1321d9c3f42c6bd56f6bf787cfba5bf3fb6b9
Parents: 4d2f684
Author: ravikiran23 <ra...@gmail.com>
Authored: Thu Jul 28 19:39:01 2016 +0530
Committer: sujith71955 <su...@gmail.com>
Committed: Thu Jul 28 19:39:01 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 5 +
.../core/load/LoadMetadataDetails.java | 20 +++
.../spark/merger/CompactionCallable.java | 36 +----
.../spark/merger/CarbonDataMergerUtil.java | 11 +-
.../execution/command/carbonTableSchema.scala | 6 +
.../spark/rdd/CarbonDataRDDFactory.scala | 15 ++-
.../org/carbondata/spark/rdd/Compactor.scala | 30 ++---
.../MajorCompactionIgnoreInMinorTest.scala | 130 +++++++++++++++++++
.../lcm/status/SegmentStatusManager.java | 2 -
9 files changed, 201 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 739462e..5852241 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -844,6 +844,11 @@ public final class CarbonCommonConstants {
*/
public static final int COMPACTION_INMEMORY_RECORD_SIZE = 120000;
+ /**
+ * If the level 2 compaction is done in minor then new compacted segment will end with .2
+ */
+ public static String LEVEL2_COMPACTION_INDEX = ".2";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/core/src/main/java/org/carbondata/core/load/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/load/LoadMetadataDetails.java b/core/src/main/java/org/carbondata/core/load/LoadMetadataDetails.java
index 0250b2e..f0b5ac9 100644
--- a/core/src/main/java/org/carbondata/core/load/LoadMetadataDetails.java
+++ b/core/src/main/java/org/carbondata/core/load/LoadMetadataDetails.java
@@ -56,6 +56,11 @@ public class LoadMetadataDetails implements Serializable {
*/
private String visibility = "true";
+ /**
+ * To know if the segment is a major compacted segment or not.
+ */
+ private String majorCompacted;
+
public String getPartitionCount() {
return partitionCount;
}
@@ -203,4 +208,19 @@ public class LoadMetadataDetails implements Serializable {
this.visibility = visibility;
}
+ /**
+ * Return true if it is a major compacted segment.
+ * @return
+ */
+ public String isMajorCompacted() {
+ return majorCompacted;
+ }
+
+ /**
+ * Set true if it is a major compacted segment.
+ * @param majorCompacted
+ */
+ public void setMajorCompacted(String majorCompacted) {
+ this.majorCompacted = majorCompacted;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java b/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java
index 7289e61..b90d46e 100644
--- a/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java
+++ b/integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java
@@ -17,51 +17,27 @@
package org.carbondata.integration.spark.merger;
-import java.util.List;
import java.util.concurrent.Callable;
-import org.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-import org.carbondata.core.load.LoadMetadataDetails;
-import org.carbondata.spark.load.CarbonLoadModel;
import org.carbondata.spark.rdd.Compactor;
-import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.execution.command.Partitioner;
+import org.apache.spark.sql.execution.command.CompactionCallableModel;
/**
- *
+ * Callable class which is used to trigger the compaction in a separate callable.
*/
public class CompactionCallable implements Callable<Void> {
- private final String hdfsStoreLocation;
- private final Partitioner partitioner;
- private final String storeLocation;
- private final CarbonTable carbonTable;
- private final String kettleHomePath;
- private final Long cubeCreationTime;
- private final List<LoadMetadataDetails> loadsToMerge;
- private final SQLContext sqlContext;
- private final CarbonLoadModel carbonLoadModel;
+ private final CompactionCallableModel compactionCallableModel;
- public CompactionCallable(String hdfsStoreLocation, CarbonLoadModel carbonLoadModel,
- Partitioner partitioner, String storeLocation, CarbonTable carbonTable, String kettleHomePath,
- Long cubeCreationTime, List<LoadMetadataDetails> loadsToMerge, SQLContext sqlContext) {
+ public CompactionCallable(CompactionCallableModel compactionCallableModel) {
- this.hdfsStoreLocation = hdfsStoreLocation;
- this.carbonLoadModel = carbonLoadModel;
- this.partitioner = partitioner;
- this.storeLocation = storeLocation;
- this.carbonTable = carbonTable;
- this.kettleHomePath = kettleHomePath;
- this.cubeCreationTime = cubeCreationTime;
- this.loadsToMerge = loadsToMerge;
- this.sqlContext = sqlContext;
+ this.compactionCallableModel = compactionCallableModel;
}
@Override public Void call() throws Exception {
- Compactor.triggerCompaction(hdfsStoreLocation, carbonLoadModel, partitioner, storeLocation,
- carbonTable, kettleHomePath, cubeCreationTime, loadsToMerge, sqlContext);
+ Compactor.triggerCompaction(compactionCallableModel);
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
index 3aa66c2..f033993 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -128,7 +128,7 @@ public final class CarbonDataMergerUtil {
public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge,
String metaDataFilepath, String MergedLoadName, CarbonLoadModel carbonLoadModel,
- String mergeLoadStartTime) {
+ String mergeLoadStartTime, CompactionType compactionType) {
boolean tableStatusUpdationStatus = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
@@ -176,6 +176,10 @@ public final class CarbonDataMergerUtil {
loadMetadataDetails.setLoadName(mergedLoadNumber);
loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
loadMetadataDetails.setPartitionCount("0");
+ // if this is a major compaction then set the segment as major compaction.
+ if (compactionType == CompactionType.MAJOR_COMPACTION) {
+ loadMetadataDetails.setMajorCompacted("true");
+ }
List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));
@@ -514,7 +518,10 @@ public final class CarbonDataMergerUtil {
// if a segment is already merged 2 levels then it s name will become .2
// need to exclude those segments from minor compaction.
- if (segName.endsWith(".2")) {
+ // if a segment is major compacted then should not be considered for minor.
+ if (segName.endsWith(CarbonCommonConstants.LEVEL2_COMPACTION_INDEX) || (
+ segment.isMajorCompacted() != null && segment.isMajorCompacted()
+ .equalsIgnoreCase("true"))) {
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 897759f..1dd066f 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -48,6 +48,7 @@ import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension
import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.datastorage.store.impl.FileFactory
+import org.carbondata.core.load.LoadMetadataDetails
import org.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.carbondata.integration.spark.merger.CompactionType
@@ -158,6 +159,11 @@ case class CompactionModel(compactionSize: Long,
carbonTable: CarbonTable,
cubeCreationTime: Long)
+case class CompactionCallableModel(hdfsStoreLocation: String, carbonLoadModel: CarbonLoadModel,
+ partitioner: Partitioner, storeLocation: String, carbonTable: CarbonTable, kettleHomePath: String,
+ cubeCreationTime: Long, loadsToMerge: util.List[LoadMetadataDetails], sqlContext: SQLContext,
+ compactionType: CompactionType)
+
object TableNewProcessor {
def apply(cm: tableModel, sqlContext: SQLContext): TableInfo = {
new TableNewProcessor(cm, sqlContext).process
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 7726739..15de7bf 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -31,7 +31,8 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileSplit
import org.apache.spark.{Logging, Partition, SparkContext, SparkEnv}
import org.apache.spark.sql.{CarbonEnv, CarbonRelation, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, Partitioner}
+import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionCallableModel,
+CompactionModel, Partitioner}
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.util.{FileUtils, SplitUtils}
@@ -476,7 +477,7 @@ object CarbonDataRDDFactory extends Logging {
}
)
- val future: Future[Void] = executor.submit(new CompactionCallable(hdfsStoreLocation,
+ val compactionCallableModel = CompactionCallableModel(hdfsStoreLocation,
carbonLoadModel,
partitioner,
storeLocation,
@@ -484,9 +485,13 @@ object CarbonDataRDDFactory extends Logging {
kettleHomePath,
compactionModel.cubeCreationTime,
loadsToMerge,
- sqlContext
- )
- )
+ sqlContext,
+ compactionModel.compactionType)
+
+ val future: Future[Void] = executor
+ .submit(new CompactionCallable(compactionCallableModel
+ )
+ )
futureList.add(future)
segList = CarbonDataMergerUtil
.filterOutAlreadyMergedSegments(segList, loadsToMerge)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
index 2680d7d..7e19ded 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
@@ -19,17 +19,14 @@ package org.carbondata.spark.rdd
import scala.collection.JavaConverters._
-import org.apache.spark.sql.SQLContext
-import org.apache.spark.sql.execution.command.{CarbonMergerMapping, Partitioner}
+import org.apache.spark.sql.execution.command.{CarbonMergerMapping, CompactionCallableModel}
import org.carbondata.common.logging.LogServiceFactory
import org.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.carbondata.core.carbon.metadata.schema.table.CarbonTable
import org.carbondata.core.constants.CarbonCommonConstants
-import org.carbondata.core.load.LoadMetadataDetails
import org.carbondata.core.util.CarbonProperties
import org.carbondata.lcm.status.SegmentStatusManager
-import org.carbondata.spark.load.{CarbonLoaderUtil, CarbonLoadModel}
+import org.carbondata.spark.load.CarbonLoaderUtil
import org.carbondata.spark.MergeResultImpl
import org.carbondata.spark.merger.CarbonDataMergerUtil
@@ -40,15 +37,18 @@ object Compactor {
val logger = LogServiceFactory.getLogService(Compactor.getClass.getName)
- def triggerCompaction(hdfsStoreLocation: String,
- carbonLoadModel: CarbonLoadModel,
- partitioner: Partitioner,
- storeLocation: String,
- carbonTable: CarbonTable,
- kettleHomePath: String,
- cubeCreationTime: Long,
- loadsToMerge: java.util.List[LoadMetadataDetails],
- sc: SQLContext): Unit = {
+ def triggerCompaction(compactionCallableModel: CompactionCallableModel): Unit = {
+
+ val hdfsStoreLocation = compactionCallableModel.hdfsStoreLocation
+ val partitioner = compactionCallableModel.partitioner
+ val storeLocation = compactionCallableModel.storeLocation
+ val carbonTable = compactionCallableModel.carbonTable
+ val kettleHomePath = compactionCallableModel.kettleHomePath
+ val cubeCreationTime = compactionCallableModel.cubeCreationTime
+ val loadsToMerge = compactionCallableModel.loadsToMerge
+ val sc = compactionCallableModel.sqlContext
+ val carbonLoadModel = compactionCallableModel.carbonLoadModel
+ val compactionType = compactionCallableModel.compactionType
val startTime = System.nanoTime();
val mergedLoadName = CarbonDataMergerUtil.getMergedLoadName(loadsToMerge)
@@ -117,7 +117,7 @@ object Compactor {
logger.info("time taken to merge " + mergedLoadName + " is " + (endTime - startTime))
CarbonDataMergerUtil
.updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath(),
- mergedLoadName, carbonLoadModel, mergeLoadStartTime
+ mergedLoadName, carbonLoadModel, mergeLoadStartTime, compactionType
)
logger
.audit("Compaction request completed for table " + carbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
new file mode 100644
index 0000000..1005e83
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -0,0 +1,130 @@
+package org.carbondata.spark.testsuite.datacompaction
+
+import java.io.File
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.util.CarbonProperties
+import org.carbondata.lcm.status.SegmentStatusManager
+import org.scalatest.BeforeAndAfterAll
+
+import scala.collection.JavaConverters._
+
+/**
+ * FT for compaction scenario where major segment should not be included in minor.
+ */
+class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ CarbonProperties.getInstance().addProperty("carbon.compaction.level.threshold", "2,2")
+ sql("drop table if exists ignoremajor")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
+ sql(
+ "CREATE TABLE IF NOT EXISTS ignoremajor (country String, ID Int, date Timestamp, name " +
+ "String, " +
+ "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" +
+ ".format'"
+ )
+
+
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv"
+
+ val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv"
+ val csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv"
+
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE ignoremajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ // compaction will happen here.
+ sql("alter table ignoremajor compact 'major'"
+ )
+ if (checkCompactionCompletedOrNot("0.1")) {
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE ignoremajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE ignoremajor OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+ sql("alter table ignoremajor compact 'minor'"
+ )
+ if (checkCompactionCompletedOrNot("2.1")) {
+ sql("alter table ignoremajor compact 'minor'"
+ )
+ }
+
+ }
+
+ }
+
+ /**
+ * Check if the compaction is completed or not.
+ *
+ * @param requiredSeg
+ * @return
+ */
+ def checkCompactionCompletedOrNot(requiredSeg: String): Boolean = {
+ var status = false
+ var noOfRetries = 0
+ while (!status && noOfRetries < 10) {
+
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", noOfRetries + "")
+ )
+ )
+ val segments = segmentStatusManager.getValidSegments().listOfValidSegments.asScala.toList
+ segments.foreach(seg =>
+ System.out.println( "valid segment is =" + seg)
+ )
+
+ if (!segments.contains(requiredSeg)) {
+ // wait for 2 seconds for compaction to complete.
+ System.out.println("sleping for 2 seconds.")
+ Thread.sleep(2000)
+ noOfRetries += 1
+ }
+ else {
+ status = true
+ }
+ }
+ return status
+ }
+
+ /**
+ * Test whether major compaction is not included in minor compaction.
+ */
+ test("delete merged folder and check segments") {
+ // delete merged segments
+ sql("clean files for table ignoremajor")
+
+ val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new
+ AbsoluteTableIdentifier(
+ CarbonProperties.getInstance.getProperty(CarbonCommonConstants.STORE_LOCATION),
+ new CarbonTableIdentifier("default", "ignoremajor", "rrr")
+ )
+ )
+ // merged segment should not be there
+ val segments = segmentStatusManager.getValidSegments.listOfValidSegments.asScala.toList
+ assert(segments.contains("0.1"))
+ assert(segments.contains("2.1"))
+ assert(!segments.contains("2"))
+ assert(!segments.contains("3"))
+
+ }
+
+ override def afterAll {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/bdc1321d/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java b/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java
index e6220d8..b35bc06 100644
--- a/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java
+++ b/processing/src/main/java/org/carbondata/lcm/status/SegmentStatusManager.java
@@ -107,7 +107,6 @@ public class SegmentStatusManager {
.getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
String dataPath = carbonTablePath.getTableStatusFilePath();
-
DataInputStream dataInputStream = null;
Gson gsonObjectToRead = new Gson();
AtomicFileOperations fileOperation =
@@ -134,7 +133,6 @@ public class SegmentStatusManager {
.equalsIgnoreCase(loadMetadataDetails.getLoadStatus())) {
// check for merged loads.
if (null != loadMetadataDetails.getMergedLoadName()) {
-
if (!listOfValidSegments.contains(loadMetadataDetails.getMergedLoadName())) {
listOfValidSegments.add(loadMetadataDetails.getMergedLoadName());
}
[24/47] incubator-carbondata git commit: [CARBONDATA-99][Bug] For
complex type filter queries issue, like and not like are failing (#870)
Posted by ra...@apache.org.
[CARBONDATA-99][Bug] For complex type filter queries issue, like and not like are failing (#870)
For complex type filter queries if query contains filter expression rather than BinaryExpression the system was not able to get the dimensions which are involved
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/82b74070
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/82b74070
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/82b74070
Branch: refs/heads/master
Commit: 82b74070b090e627c333f1a574111c33e281725e
Parents: f6e9fbc
Author: sujith71955 <su...@gmail.com>
Authored: Mon Jul 25 23:24:00 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 23:24:00 2016 +0530
----------------------------------------------------------------------
.../carbondata/query/carbon/executor/util/QueryUtil.java | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/82b74070/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index b1de455..d36238e 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -70,7 +70,6 @@ import org.carbondata.query.complex.querytypes.PrimitiveQueryType;
import org.carbondata.query.complex.querytypes.StructQueryType;
import org.carbondata.query.expression.ColumnExpression;
import org.carbondata.query.expression.Expression;
-import org.carbondata.query.expression.logical.BinaryLogicalExpression;
import org.carbondata.query.filter.resolver.FilterResolverIntf;
import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -1105,12 +1104,7 @@ public class QueryUtil {
}
List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>();
Expression filterExpression = filterResolverTree.getFilterExpression();
- if (filterExpression instanceof BinaryLogicalExpression) {
- BinaryLogicalExpression logicalExpression = (BinaryLogicalExpression) filterExpression;
- dimensionResolvedInfos.addAll(logicalExpression.getColumnList());
- } else {
- addColumnDimensions(filterExpression, filterDimensions);
- }
+ addColumnDimensions(filterExpression, filterDimensions);
for (ColumnExpression info : dimensionResolvedInfos) {
if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
filterDimensions.add(info.getDimension());
@@ -1130,7 +1124,8 @@ public class QueryUtil {
*/
private static void addColumnDimensions(Expression expression,
Set<CarbonDimension> filterDimensions) {
- if (null != expression && expression instanceof ColumnExpression) {
+ if (null != expression && expression instanceof ColumnExpression
+ && ((ColumnExpression) expression).isDimension()) {
filterDimensions.add(((ColumnExpression) expression).getDimension());
return;
}
[23/47] incubator-carbondata git commit: [CARBONDATA-106] Added audit
logs for create and load command failures (#868)
Posted by ra...@apache.org.
[CARBONDATA-106] Added audit logs for create and load command failures (#868)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f6e9fbc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f6e9fbc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f6e9fbc9
Branch: refs/heads/master
Commit: f6e9fbc9b892d327abafbf71d57c088bb66c8041
Parents: ad1c985
Author: Manu <ma...@gmail.com>
Authored: Mon Jul 25 21:28:05 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Mon Jul 25 21:28:05 2016 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonSqlParser.scala | 201 ++++++++++---------
.../execution/command/carbonTableSchema.scala | 8 +
.../scala/org/apache/spark/util/FileUtils.scala | 6 +-
.../TestLoadDataWithNotProperInputFile.scala | 6 +-
4 files changed, 124 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f6e9fbc9/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index e222bec..fdfc683 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -36,9 +36,11 @@ import org.apache.spark.sql.execution.command.{DimensionRelation, _}
import org.apache.spark.sql.execution.datasources.DescribeCommand
import org.apache.spark.sql.hive.HiveQlWrapper
+import org.carbondata.common.logging.LogServiceFactory
import org.carbondata.core.carbon.metadata.datatype.DataType
import org.carbondata.core.constants.CarbonCommonConstants
import org.carbondata.core.util.DataTypeUtil
+import org.carbondata.processing.etl.DataLoadingException
import org.carbondata.spark.exception.MalformedCarbonCommandException
import org.carbondata.spark.util.CommonUtil
@@ -48,6 +50,7 @@ import org.carbondata.spark.util.CommonUtil
class CarbonSqlParser()
extends AbstractSparkSQLParser with Logging {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
protected val AGGREGATE = carbonKeyWord("AGGREGATE")
protected val AS = carbonKeyWord("AS")
protected val AGGREGATION = carbonKeyWord("AGGREGATION")
@@ -333,108 +336,124 @@ class CarbonSqlParser()
// if create table taken is found then only we will handle.
case Token("TOK_CREATETABLE", children) =>
- var fields: Seq[Field] = Seq[Field]()
- var tableComment: String = ""
- var tableProperties = Map[String, String]()
- var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
- var likeTableName: String = ""
- var storedBy: String = ""
- var ifNotExistPresent: Boolean = false
- var dbName: Option[String] = None
- var tableName: String = ""
- children.collect {
- // collecting all the field list
- case list@Token("TOK_TABCOLLIST", _) =>
- val cols = BaseSemanticAnalyzer.getColumns(list, true)
- if (cols != null) {
- val dupColsGrp = cols.asScala
- .groupBy(x => x.getName) filter { case (_, colList) => colList
- .size > 1
- }
- if (dupColsGrp.size > 0) {
- var columnName: String = ""
- dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
- columnName = columnName.substring(0, columnName.lastIndexOf(", "))
- val errorMessage = "Duplicate column name: " + columnName + " found in table " +
- ".Please check create table statement."
- throw new MalformedCarbonCommandException(errorMessage)
- }
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val name = Option(col.getName())
- // This is to parse complex data types
- val x = col.getName + ' ' + col.getType
- val f: Field = anyFieldDef(new lexical.Scanner(x))
- match {
- case Success(field, _) => field
- case failureOrError => new Field(columnName, dataType, name, None, null,
- Some("columnar"))
+ var fields: Seq[Field] = Seq[Field]()
+ var tableComment: String = ""
+ var tableProperties = Map[String, String]()
+ var partitionCols: Seq[PartitionerField] = Seq[PartitionerField]()
+ var likeTableName: String = ""
+ var storedBy: String = ""
+ var ifNotExistPresent: Boolean = false
+ var dbName: Option[String] = None
+ var tableName: String = ""
+
+ try {
+
+ children.collect {
+ // collecting all the field list
+ case list@Token("TOK_TABCOLLIST", _) =>
+ val cols = BaseSemanticAnalyzer.getColumns(list, true)
+ if (cols != null) {
+ val dupColsGrp = cols.asScala
+ .groupBy(x => x.getName) filter { case (_, colList) => colList
+ .size >
+ 1
+ }
+ if (dupColsGrp.size > 0) {
+ var columnName: String = ""
+ dupColsGrp.toSeq.foreach(columnName += _._1 + ", ")
+ columnName = columnName.substring(0, columnName.lastIndexOf(", "))
+ val errorMessage = "Duplicate column name: " + columnName + " found in table " +
+ ".Please check create table statement."
+ throw new MalformedCarbonCommandException(errorMessage)
}
- // the data type of the decimal type will be like decimal(10,0)
- // so checking the start of the string and taking the precision and scale.
- // resetting the data type with decimal
- if (f.dataType.getOrElse("").startsWith("decimal")) {
- val (precision, scale) = getScaleAndPrecision(col.getType)
- f.precision = precision
- f.scale = scale
- f.dataType = Some("decimal")
+ cols.asScala.map { col =>
+ val columnName = col.getName()
+ val dataType = Option(col.getType)
+ val name = Option(col.getName())
+ // This is to parse complex data types
+ val x = col.getName + ' ' + col.getType
+ val f: Field = anyFieldDef(new lexical.Scanner(x))
+ match {
+ case Success(field, _) => field
+ case failureOrError => new Field(columnName, dataType, name, None, null,
+ Some("columnar"))
+ }
+ // the data type of the decimal type will be like decimal(10,0)
+ // so checking the start of the string and taking the precision and scale.
+ // resetting the data type with decimal
+ if (f.dataType.getOrElse("").startsWith("decimal")) {
+ val (precision, scale) = getScaleAndPrecision(col.getType)
+ f.precision = precision
+ f.scale = scale
+ f.dataType = Some("decimal")
+ }
+ fields ++= Seq(f)
}
- fields ++= Seq(f)
}
- }
-
- case Token("TOK_IFNOTEXISTS", _) =>
- ifNotExistPresent = true
- case t@Token("TOK_TABNAME", _) =>
- val (db, tblName) = extractDbNameTableName(t)
- dbName = db
- tableName = tblName.toLowerCase()
-
- case Token("TOK_TABLECOMMENT", child :: Nil) =>
- tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
-
- case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
- val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
- if (cols != null) {
- cols.asScala.map { col =>
- val columnName = col.getName()
- val dataType = Option(col.getType)
- val comment = col.getComment
- val partitionCol = new PartitionerField(columnName, dataType, comment)
- partitionCols ++= Seq(partitionCol)
+ case Token("TOK_IFNOTEXISTS", _) =>
+ ifNotExistPresent = true
+
+ case t@Token("TOK_TABNAME", _) =>
+ val (db, tblName) = extractDbNameTableName(t)
+ dbName = db
+ tableName = tblName.toLowerCase()
+
+ case Token("TOK_TABLECOMMENT", child :: Nil) =>
+ tableComment = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
+
+ case Token("TOK_TABLEPARTCOLS", list@Token("TOK_TABCOLLIST", _) :: Nil) =>
+ val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
+ if (cols != null) {
+ cols.asScala.map { col =>
+ val columnName = col.getName()
+ val dataType = Option(col.getType)
+ val comment = col.getComment
+ val partitionCol = new PartitionerField(columnName, dataType, comment)
+ partitionCols ++= Seq(partitionCol)
+ }
}
- }
- case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
- tableProperties ++= getProperties(list)
+ case Token("TOK_TABLEPROPERTIES", list :: Nil) =>
+ tableProperties ++= getProperties(list)
- case Token("TOK_LIKETABLE", child :: Nil) =>
- likeTableName = child.getChild(0).getText()
+ case Token("TOK_LIKETABLE", child :: Nil) =>
+ likeTableName = child.getChild(0).getText()
- case Token("TOK_STORAGEHANDLER", child :: Nil) =>
- storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
+ case Token("TOK_STORAGEHANDLER", child :: Nil) =>
+ storedBy = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
- case _ => // Unsupport features
- }
-
- if (!storedBy.equals(CarbonContext.datasourceName)) {
- // TODO: should execute by Hive instead of error
- sys.error("Not a carbon format request")
- }
+ case _ => // Unsupport features
+ }
- // validate tblProperties
- if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
- throw new MalformedCarbonCommandException("Invalid table properties")
- }
- // prepare table model of the collected tokens
- val tableModel: tableModel = prepareTableModel(ifNotExistPresent, dbName, tableName, fields,
- partitionCols,
- tableProperties)
+ if (!storedBy.equals(CarbonContext.datasourceName)) {
+ // TODO: should execute by Hive instead of error
+ sys.error("Not a carbon format request")
+ }
- // get logical plan.
- CreateTable(tableModel)
+ // validate tblProperties
+ if (!CommonUtil.validateTblProperties(tableProperties, fields)) {
+ throw new MalformedCarbonCommandException("Invalid table properties")
+ }
+ // prepare table model of the collected tokens
+ val tableModel: tableModel = prepareTableModel(ifNotExistPresent,
+ dbName,
+ tableName,
+ fields,
+ partitionCols,
+ tableProperties)
+
+ // get logical plan.
+ CreateTable(tableModel)
+ }
+ catch {
+ case ce: MalformedCarbonCommandException =>
+ val message = if (tableName.isEmpty) "Create table command failed. "
+ else if (!dbName.isDefined) s"Create table command failed for $tableName. "
+ else s"Create table command failed for ${dbName.get}.$tableName. "
+ LOGGER.audit(message + ce.getMessage)
+ throw ce
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f6e9fbc9/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 5da01ac..897759f 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -52,6 +52,7 @@ import org.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.carbondata.integration.spark.merger.CompactionType
import org.carbondata.lcm.status.SegmentStatusManager
+import org.carbondata.processing.etl.DataLoadingException
import org.carbondata.spark.exception.MalformedCarbonCommandException
import org.carbondata.spark.load._
import org.carbondata.spark.partition.api.impl.QueryPartitionHelper
@@ -1626,6 +1627,13 @@ private[sql] case class LoadCube(
}
}
+ } catch {
+ case dle: DataLoadingException =>
+ LOGGER.audit(s"Dataload failed for $schemaName.$tableName. " + dle.getMessage)
+ throw dle
+ case mce: MalformedCarbonCommandException =>
+ LOGGER.audit(s"Dataload failed for $schemaName.$tableName. " + mce.getMessage)
+ throw mce
} finally {
if (carbonLock != null) {
if (carbonLock.unlock()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f6e9fbc9/integration/spark/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/util/FileUtils.scala b/integration/spark/src/main/scala/org/apache/spark/util/FileUtils.scala
index 8344956..0fd43a3 100644
--- a/integration/spark/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -57,7 +57,7 @@ object FileUtils extends Logging {
*/
def getPaths(inputPath: String): String = {
if (inputPath == null || inputPath.isEmpty) {
- throw new DataLoadingException("input file path cannot be empty.")
+ throw new DataLoadingException("Input file path cannot be empty.")
} else {
val stringBuild = new StringBuilder()
val filePaths = inputPath.split(",")
@@ -65,14 +65,14 @@ object FileUtils extends Logging {
val fileType = FileFactory.getFileType(filePaths(i))
val carbonFile = FileFactory.getCarbonFile(filePaths(i), fileType)
if (!carbonFile.exists()) {
- throw new DataLoadingException(s"the input file does not exist: ${filePaths(i)}" )
+ throw new DataLoadingException(s"The input file does not exist: ${filePaths(i)}" )
}
getPathsFromCarbonFile(carbonFile, stringBuild)
}
if (stringBuild.nonEmpty) {
stringBuild.substring(0, stringBuild.size - 1)
} else {
- throw new DataLoadingException("please check your input path and make sure " +
+ throw new DataLoadingException("Please check your input path and make sure " +
"that files end with '.csv' and content is not empty.")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f6e9fbc9/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
index b5830c4..9751f0b 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNotProperInputFile.scala
@@ -44,7 +44,7 @@ class TestLoadDataWithNotProperInputFile extends QueryTest {
GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
} catch {
case e: Throwable =>
- assert(e.getMessage.contains("please check your input path and make sure " +
+ assert(e.getMessage.contains("Please check your input path and make sure " +
"that files end with '.csv' and content is not empty"))
}
}
@@ -58,7 +58,7 @@ class TestLoadDataWithNotProperInputFile extends QueryTest {
GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
} catch {
case e: Throwable =>
- assert(e.getMessage.contains("please check your input path and make sure " +
+ assert(e.getMessage.contains("Please check your input path and make sure " +
"that files end with '.csv' and content is not empty"))
}
}
@@ -72,7 +72,7 @@ class TestLoadDataWithNotProperInputFile extends QueryTest {
GlobalDictionaryUtil.loadDataFrame(CarbonHiveContext, carbonLoadModel)
} catch {
case e: Throwable =>
- assert(e.getMessage.contains("the input file does not exist"))
+ assert(e.getMessage.contains("The input file does not exist"))
}
}
}
[45/47] incubator-carbondata git commit: Merge remote-tracking branch
'carbon_master/master' into apache/master
Posted by ra...@apache.org.
Merge remote-tracking branch 'carbon_master/master' into apache/master
Conflicts:
core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
core/src/main/java/org/carbondata/query/aggregator/impl/CountAggregator.java
core/src/main/java/org/carbondata/query/aggregator/util/MeasureAggregatorFactory.java
core/src/main/java/org/carbondata/query/carbon/result/iterator/AbstractDetailQueryResultIterator.java
core/src/main/java/org/carbondata/query/carbon/result/iterator/DetailQueryResultIterator.java
core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
core/src/main/java/org/carbondata/scan/filter/resolver/AndFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
core/src/main/java/org/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
integration/spark/src/main/java/org/carbondata/integration/spark/merger/CompactionCallable.java
integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonMergerRDD.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonScanRDD.scala
integration/spark/src/main/scala/org/carbondata/spark/rdd/Compactor.scala
integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
processing/src/main/java/org/carbondata/processing/mdkeygen/MDKeyGenStep.java
processing/src/main/java/org/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
processing/src/main/java/org/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/50dfdf6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/50dfdf6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/50dfdf6c
Branch: refs/heads/master
Commit: 50dfdf6c8eea672665fe99c5ac4603107a435209
Parents: 10ed89a b82a960
Author: ravipesala <ra...@gmail.com>
Authored: Mon Aug 1 12:58:17 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Aug 1 12:58:17 2016 +0530
----------------------------------------------------------------------
.../core/carbon/ColumnIdentifier.java | 4 +
.../carbon/datastore/SegmentTaskIndexStore.java | 2 +-
.../metadata/schema/table/CarbonTable.java | 48 ++--
.../schema/table/column/CarbonDimension.java | 4 +
.../core/carbon/path/CarbonStorePath.java | 2 +-
.../core/carbon/path/CarbonTablePath.java | 18 +-
.../carbon/querystatistics/QueryStatistic.java | 12 +
.../core/constants/CarbonCommonConstants.java | 29 ++
.../org/carbondata/core/load/BlockDetails.java | 8 +-
.../core/load/LoadMetadataDetails.java | 20 ++
.../core/util/CarbonLoadStatisticsDummy.java | 12 +-
.../core/util/CarbonLoadStatisticsImpl.java | 65 ++---
.../org/carbondata/core/util/DataTypeUtil.java | 67 +++++
.../carbondata/core/util/LoadStatistics.java | 6 +-
.../aggregator/impl/CountStarAggregator.java | 51 ++++
.../scan/executor/util/QueryUtil.java | 54 ++--
.../scan/filter/FilterExpressionProcessor.java | 19 +-
.../org/carbondata/scan/filter/FilterUtil.java | 32 +++
.../filter/resolver/AndFilterResolverImpl.java | 4 +-
.../resolver/LogicalFilterResolverImpl.java | 10 +-
.../visitor/CustomTypeDictionaryVisitor.java | 8 +
.../visitor/DictionaryColumnVisitor.java | 4 +-
.../examples/ComplexTypeExample.scala | 6 +-
.../hadoop/test/util/StoreCreator.java | 2 +-
.../spark/sql/common/util/CsvCompare.scala | 1 -
.../spark/merger/CompactionCallable.java | 36 +--
.../carbondata/spark/load/CarbonLoaderUtil.java | 22 +-
.../spark/merger/CarbonDataMergerUtil.java | 84 ++++--
.../spark/sql/CarbonDatasourceRelation.scala | 24 +-
.../org/apache/spark/sql/CarbonSqlParser.scala | 216 ++++++++-------
.../spark/sql/SparkUnknownExpression.scala | 24 +-
.../execution/command/carbonTableSchema.scala | 32 ++-
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 27 +-
.../scala/org/apache/spark/util/FileUtils.scala | 6 +-
.../org/apache/spark/util/SplitUtils.scala | 6 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 65 +++--
.../spark/rdd/CarbonDataRDDFactory.scala | 264 ++++++++++---------
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 37 ++-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 183 +++++++------
.../carbondata/spark/rdd/CarbonScanRDD.scala | 51 +++-
.../org/carbondata/spark/rdd/Compactor.scala | 79 +++---
.../spark/tasks/DictionaryWriterTask.scala | 56 ++--
.../carbondata/spark/util/CarbonScalaUtil.scala | 13 +
.../spark/util/GlobalDictionaryUtil.scala | 16 +-
integration/spark/src/test/resources/array1.csv | 2 +
.../resources/complexTypeDecimalNestedHive.csv | 8 +
.../spark/src/test/resources/datadelimiter.csv | 11 +
.../src/test/resources/datanullmeasurecol.csv | 3 +
.../spark/src/test/resources/datasample.csv | 1 +
.../resources/decimalBoundaryDataCarbon.csv | 12 +
.../test/resources/decimalBoundaryDataHive.csv | 11 +
.../test/resources/emptyDimensionDataHive.csv | 20 ++
.../spark/src/test/resources/struct_all.csv | 4 +
.../src/test/resources/structusingstruct.csv | 2 +
.../spark/src/test/resources/timestampdata.csv | 2 +
.../apache/spark/sql/TestCarbonSqlParser.scala | 72 ++---
...plexPrimitiveTimestampDirectDictionary.scala | 12 +
.../complexType/TestComplexTypeQuery.scala | 52 +++-
.../dataload/TestLoadDataWithJunkChars.scala | 43 +++
.../dataload/TestLoadDataWithNullMeasures.scala | 1 +
.../testsuite/bigdecimal/TestBigDecimal.scala | 52 +++-
.../createtable/TestCreateTableSyntax.scala | 29 +-
.../DataCompactionCardinalityBoundryTest.scala | 6 +-
.../datacompaction/DataCompactionTest.scala | 6 +-
.../MajorCompactionIgnoreInMinorTest.scala | 130 +++++++++
.../MajorCompactionStopsAfterCompaction.scala | 125 +++++++++
.../TestDataWithDicExcludeAndInclude.scala | 22 +-
.../dataload/TestLoadDataWithHiveSyntax.scala | 59 ++++-
.../dataload/TestLoadDataWithNoMeasure.scala | 2 +-
.../TestLoadDataWithNotProperInputFile.scala | 6 +-
.../NoDictionaryColumnTestCase.scala | 6 +-
...estampDataTypeDirectDictionaryTestCase.scala | 49 +++-
...TypeDirectDictionaryWithNoDictTestCase.scala | 6 +-
.../TimestampDataTypeNullDataTest.scala | 2 +-
.../filterexpr/AllDataTypesTestCaseFilter.scala | 14 +-
.../filterexpr/CountStarTestCase.scala | 72 +++++
.../NullMeasureValueTestCaseFilter.scala | 7 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 2 +-
.../util/GlobalDictionaryUtilTestCase.scala | 4 +-
.../carbondata/lcm/locks/CarbonLockFactory.java | 27 +-
.../org/carbondata/lcm/locks/HdfsFileLock.java | 25 +-
.../org/carbondata/lcm/locks/LocalFileLock.java | 16 +-
.../carbondata/lcm/locks/ZooKeeperLocking.java | 49 +++-
.../lcm/status/SegmentStatusManager.java | 2 -
.../processing/csvload/GraphExecutionUtil.java | 31 +++
.../csvreaderstep/CustomDataStream.java | 108 ++++++++
.../processing/csvreaderstep/CustomReader.java | 157 -----------
.../csvreaderstep/UnivocityCsvParser.java | 13 +-
.../processing/datatypes/PrimitiveDataType.java | 28 +-
.../processing/mdkeygen/MDKeyGenStep.java | 2 +-
.../processing/mdkeygen/MDKeyGenStepMeta.java | 3 +-
.../sortandgroupby/sortdata/SortDataRows.java | 2 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 21 ++
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 3 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 72 ++++-
.../FileStoreSurrogateKeyGenForCSV.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 7 +-
.../processing/util/CarbonSchemaParser.java | 1 +
98 files changed, 2240 insertions(+), 914 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
index 765dc60,d0037e1..995bf17
--- a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
@@@ -285,61 -155,90 +285,128 @@@ public final class DataTypeUtil
return null;
}
try {
+ switch (dataType) {
+ case DOUBLE:
+ return data;
+ case LONG:
+ return data;
+ case DECIMAL:
+ java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data.toString());
+ scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal);
+ org.apache.spark.sql.types.Decimal decConverter =
+ new org.apache.spark.sql.types.Decimal();
+ return decConverter.set(scalaDecVal);
+ default:
+ return data;
+ }
+ } catch (NumberFormatException ex) {
+ LOGGER.error("Problem while converting data type" + data);
+ return null;
+ }
+
+ }
+
+ /**
+ * This method will parse a given string value corresponding to its datatype
+ *
+ * @param value value to parse
+ * @param dataType datatype for that value
+ * @return
+ */
+ public static boolean validateColumnValueForItsDataType(String value, DataType dataType) {
+ try {
Object parsedValue = null;
- switch (actualDataType) {
+ // validation will not be done for timestamp datatype as for timestamp direct dictionary
+ // is generated. No dictionary file is created for timestamp datatype column
+ switch (dataType) {
+ case DECIMAL:
+ parsedValue = new BigDecimal(value);
+ break;
case INT:
- parsedValue = Integer.parseInt(data);
+ parsedValue = Integer.parseInt(value);
break;
case LONG:
- parsedValue = Long.parseLong(data);
+ parsedValue = Long.valueOf(value);
+ break;
+ case DOUBLE:
+ parsedValue = Double.valueOf(value);
break;
default:
- return data;
+ return true;
}
- if(null != parsedValue) {
- return data;
+ if (null != parsedValue) {
+ return true;
}
- return null;
- } catch (NumberFormatException ex) {
- return null;
+ return false;
+ } catch (Exception e) {
+ return false;
}
}
+
+ /**
+ * This method will parse a given string value corresponding to its data type
+ *
+ * @param value value to parse
+ * @param dimension dimension to get data type and precision and scale in case of decimal
+ * data type
+ * @return
+ */
+ public static String normalizeColumnValueForItsDataType(String value, CarbonDimension dimension) {
+ try {
+ Object parsedValue = null;
+ // validation will not be done for timestamp datatype as for timestamp direct dictionary
+ // is generated. No dictionary file is created for timestamp datatype column
+ switch (dimension.getDataType()) {
+ case DECIMAL:
+ return parseStringToBigDecimal(value, dimension);
+ case INT:
+ case LONG:
+ parsedValue = normalizeIntAndLongValues(value, dimension.getDataType());
+ break;
+ case DOUBLE:
+ parsedValue = Double.parseDouble(value);
+ break;
+ default:
+ return value;
+ }
+ if (null != parsedValue) {
+ return value;
+ }
+ return null;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ /**
+ * This method will parse a value to its datatype if datatype is decimal else will return
+ * the value passed
+ *
+ * @param value value to be parsed
+ * @param dimension
+ * @return
+ */
+ public static String parseValue(String value, CarbonDimension dimension) {
+ try {
+ switch (dimension.getDataType()) {
+ case DECIMAL:
+ return parseStringToBigDecimal(value, dimension);
+ default:
+ return value;
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static String parseStringToBigDecimal(String value, CarbonDimension dimension) {
+ BigDecimal bigDecimal = new BigDecimal(value)
+ .setScale(dimension.getColumnSchema().getScale(), RoundingMode.HALF_UP);
+ BigDecimal normalizedValue =
+ normalizeDecimalValue(bigDecimal, dimension.getColumnSchema().getPrecision());
+ if (null != normalizedValue) {
+ return normalizedValue.toString();
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
index 3b978d7,0000000..8fee45a
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/scan/executor/util/QueryUtil.java
@@@ -1,931 -1,0 +1,947 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.executor.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.carbondata.core.cache.Cache;
+import org.carbondata.core.cache.CacheProvider;
+import org.carbondata.core.cache.CacheType;
+import org.carbondata.core.cache.dictionary.Dictionary;
+import org.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.CarbonTableIdentifier;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.metadata.CarbonMetadata;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.CarbonTable;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
+import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.keygenerator.KeyGenerator;
+import org.carbondata.core.util.CarbonUtil;
+import org.carbondata.core.util.CarbonUtilException;
+import org.carbondata.scan.complextypes.ArrayQueryType;
+import org.carbondata.scan.complextypes.PrimitiveQueryType;
+import org.carbondata.scan.complextypes.StructQueryType;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.carbondata.scan.expression.ColumnExpression;
+import org.carbondata.scan.expression.Expression;
+import org.carbondata.scan.expression.logical.BinaryLogicalExpression;
+import org.carbondata.scan.filter.GenericQueryType;
+import org.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.carbondata.scan.model.QueryDimension;
+import org.carbondata.scan.model.QueryMeasure;
+import org.carbondata.scan.model.QueryModel;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * Utility class for query execution
+ */
+public class QueryUtil {
+
+ /**
+ * Below method will be used to get the masked byte range based on the query
+ * dimension. It will give the range in the mdkey. This will be used to get
+ * the actual key array from masked mdkey
+ *
+ * @param queryDimensions query dimension selected in query
+ * @param keyGenerator key generator
+ * @return masked key
+ */
+ public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions,
+ KeyGenerator keyGenerator) {
+ Set<Integer> byteRangeSet = new TreeSet<Integer>();
+ int[] byteRange = null;
+ for (int i = 0; i < queryDimensions.size(); i++) {
+
+ // as no dictionary column and complex type columns
+ // are not selected in the mdkey
+ // so we will not select the those dimension for calculating the
+ // range
+ if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
+ continue;
+ }
+ // get the offset of the dimension in the mdkey
+ byteRange =
+ keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal());
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ byteRangeSet.add(j);
+ }
+ }
+ int[] maksedByteRange = new int[byteRangeSet.size()];
+ int index = 0;
+ Iterator<Integer> iterator = byteRangeSet.iterator();
+ // add the masked byte range
+ while (iterator.hasNext()) {
+ maksedByteRange[index++] = iterator.next();
+ }
+ return maksedByteRange;
+ }
+
+ public static int[] getMaskedByteRangeBasedOrdinal(List<Integer> ordinals,
+ KeyGenerator keyGenerator) {
+ Set<Integer> byteRangeSet = new TreeSet<Integer>();
+ int[] byteRange = null;
+ for (int i = 0; i < ordinals.size(); i++) {
+
+ // get the offset of the dimension in the mdkey
+ byteRange = keyGenerator.getKeyByteOffsets(ordinals.get(i));
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ byteRangeSet.add(j);
+ }
+ }
+ int[] maksedByteRange = new int[byteRangeSet.size()];
+ int index = 0;
+ Iterator<Integer> iterator = byteRangeSet.iterator();
+ // add the masked byte range
+ while (iterator.hasNext()) {
+ maksedByteRange[index++] = iterator.next();
+ }
+ return maksedByteRange;
+ }
+
+ /**
+ * Below method will return the max key based on the dimension ordinal
+ *
+ * @param keyOrdinalList
+ * @param generator
+ * @return
+ * @throws KeyGenException
+ */
+ public static byte[] getMaxKeyBasedOnOrinal(List<Integer> keyOrdinalList, KeyGenerator generator)
+ throws KeyGenException {
+ long[] max = new long[generator.getDimCount()];
+ Arrays.fill(max, 0L);
+
+ for (int i = 0; i < keyOrdinalList.size(); i++) {
+ // adding for dimension which is selected in query
+ max[keyOrdinalList.get(i)] = Long.MAX_VALUE;
+ }
+ return generator.generateKey(max);
+ }
+
+ /**
+ * To get the max key based on dimensions. i.e. all other dimensions will be
+ * set to 0 bits and the required query dimension will be masked with all
+ * LONG.MAX so that we can mask key and then compare while aggregating This
+ * can be useful during filter query when only few dimensions were selected
+ * out of row group
+ *
+ * @param queryDimensions dimension selected in query
+ * @param generator key generator
+ * @return max key for dimension
+ * @throws KeyGenException if any problem while generating the key
+ */
+ public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions,
+ KeyGenerator generator) throws KeyGenException {
+ long[] max = new long[generator.getDimCount()];
+ Arrays.fill(max, 0L);
+
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ // as no dictionary column and complex type columns
+ // are not selected in the mdkey
+ // so we will not select the those dimension for calculating the
+ // range
+ if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
+ continue;
+ }
+ // adding for dimension which is selected in query
+ max[queryDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE;
+ }
+
+ return generator.generateKey(max);
+ }
+
+ /**
+ * Below method will be used to get the masked key for query
+ *
+ * @param keySize size of the masked key
+ * @param maskedKeyRanges masked byte range
+ * @return masked bytes
+ */
+ public static int[] getMaskedByte(int keySize, int[] maskedKeyRanges) {
+ int[] maskedKey = new int[keySize];
+ // all the non selected dimension will be filled with -1
+ Arrays.fill(maskedKey, -1);
+ for (int i = 0; i < maskedKeyRanges.length; i++) {
+ maskedKey[maskedKeyRanges[i]] = i;
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method will be used to get the dimension block index in file based
+ * on query dimension
+ *
+ * @param queryDimensions query dimension
+ * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension
+ * @return block index of file
+ */
+ public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
+ Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+ List<CarbonDimension> customAggregationDimension) {
+ // using set as in row group columns will point to same block
+ Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
+ int blockIndex = 0;
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ blockIndex =
+ dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
+ dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, queryDimensions.get(i).getDimension());
++ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
++ addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
++ }
+ }
+ for (int i = 0; i < customAggregationDimension.size(); i++) {
+ blockIndex =
+ dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
++ // not adding the children dimension as dimension aggregation
++ // is not push down in case of complex dimension
+ dimensionBlockIndex.add(blockIndex);
- addChildrenBlockIndex(blockIndex, dimensionBlockIndex, customAggregationDimension.get(i));
+ }
+ return ArrayUtils
+ .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
+ }
+
+ /**
+ * Below method will be used to add the children block index
+ * this will be basically for complex dimension which will have children
+ *
- * @param startBlockIndex start block index
- * @param blockIndexList block index list
- * @param dimension parent dimension
++ * @param blockIndexes block indexes
++ * @param dimension parent dimension
+ */
- private static void addChildrenBlockIndex(int startBlockIndex, Set<Integer> blockIndexList,
- CarbonDimension dimension) {
++ private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) {
+ for (int i = 0; i < dimension.numberOfChild(); i++) {
- blockIndexList.add(++startBlockIndex);
- addChildrenBlockIndex(startBlockIndex, blockIndexList,
- dimension.getListOfChildDimensions().get(i));
++ addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
++ blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
+ }
+ }
+
+ /**
+ * Below method will be used to get the dictionary mapping for all the
+ * dictionary encoded dimension present in the query
+ *
+ * @param queryDimensions query dimension present in the query this will be used to
+ * convert the result from surrogate key to actual data
+ * @param absoluteTableIdentifier absolute table identifier
+ * @return dimension unique id to its dictionary map
+ * @throws QueryExecutionException
+ */
+ public static Map<String, Dictionary> getDimensionDictionaryDetail(
+ List<QueryDimension> queryDimensions,
+ Set<CarbonDimension> filterComplexDimensions,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws QueryExecutionException {
+ // to store dimension unique column id list, this is required as
+ // dimension can be present in
+ // query dimension, as well as some aggregation function will be applied
+ // in the same dimension
+ // so we need to get only one instance of dictionary
+ // direct dictionary skip is done only for the dictionary lookup
+ Set<String> dictionaryDimensionFromQuery = new HashSet<String>();
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ List<Encoding> encodingList = queryDimensions.get(i).getDimension().getEncoder();
+ // TODO need to remove the data type check for parent column in complex type no need to
+ // write encoding dictionary
+ if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil
+ .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+
+ if (queryDimensions.get(i).getDimension().numberOfChild() == 0) {
+ dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId());
+ }
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(),
+ dictionaryDimensionFromQuery);
+ }
+ }
+ }
+ Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator();
+ while (iterator.hasNext()) {
+ getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery);
+ }
+ // converting to list as api exposed needed list which i think
+ // is not correct
+ List<String> dictionaryColumnIdList =
+ new ArrayList<String>(dictionaryDimensionFromQuery.size());
+ dictionaryColumnIdList.addAll(dictionaryDimensionFromQuery);
+ return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier);
+ }
+
+ /**
+ * Below method will be used to fill the children dimension column id
+ *
+ * @param queryDimensions query dimension
+ * @param dictionaryDimensionFromQuery dictionary dimension for query
+ */
+ private static void getChildDimensionDictionaryDetail(CarbonDimension queryDimensions,
+ Set<String> dictionaryDimensionFromQuery) {
+ for (int j = 0; j < queryDimensions.numberOfChild(); j++) {
+ List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
+ if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
+ getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
+ dictionaryDimensionFromQuery);
- } else if(!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
++ } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ dictionaryDimensionFromQuery
+ .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to get the column id to its dictionary mapping
+ *
+ * @param dictionaryColumnIdList dictionary column list
+ * @param absoluteTableIdentifier absolute table identifier
+ * @return dictionary mapping
+ * @throws QueryExecutionException
+ */
+ private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws QueryExecutionException {
+ // this for dictionary unique identifier
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
+ getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,
+ absoluteTableIdentifier.getCarbonTableIdentifier());
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ Cache forwardDictionaryCache = cacheProvider
+ .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
+ List<Dictionary> columnDictionaryList = null;
+ try {
+ columnDictionaryList = forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
+ } catch (CarbonUtilException e) {
+ throw new QueryExecutionException(e);
+ }
+ Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size());
+ for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
+ // TODO: null check for column dictionary, if cache size is less it
+ // might return null here, in that case throw exception
+ columnDictionaryMap.put(dictionaryColumnIdList.get(i), columnDictionaryList.get(i));
+ }
+ return columnDictionaryMap;
+ }
+
+ /**
+ * Below method will be used to get the dictionary column unique identifier
+ *
+ * @param dictionaryColumnIdList dictionary
+ * @param carbonTableIdentifier
+ * @return
+ */
+ private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(
+ List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier)
+ throws QueryExecutionException {
+ CarbonTable carbonTable =
+ CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName());
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
+ new ArrayList<>(dictionaryColumnIdList.size());
+ for (String columnId : dictionaryColumnIdList) {
+ CarbonDimension dimension = CarbonMetadata.getInstance()
+ .getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
+ if (null == dimension) {
+ throw new QueryExecutionException("The column id " + columnId + " could not be resolved.");
+ }
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
+ new DictionaryColumnUniqueIdentifier(carbonTableIdentifier,
+ dimension.getColumnIdentifier(), dimension.getDataType());
+ dictionaryColumnUniqueIdentifiers.add(dictionaryColumnUniqueIdentifier);
+ }
+ return dictionaryColumnUniqueIdentifiers;
+ }
+
+ /**
+ * Below method will used to get the method will be used to get the measure
+ * block indexes to be read from the file
+ *
+ * @param queryMeasures query measure
+ * @param expressionMeasure measure present in the expression
+ * @param ordinalToBlockIndexMapping measure ordinal to block mapping
+ * @return block indexes
+ */
+ public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
+ List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping) {
+ Set<Integer> measureBlockIndex = new HashSet<Integer>();
+ for (int i = 0; i < queryMeasures.size(); i++) {
+ measureBlockIndex
+ .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
+ }
+ for (int i = 0; i < expressionMeasure.size(); i++) {
+ measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+ }
+ return ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
+ }
+
+ /**
+ * Below method will be used to get the masked byte range for dimension
+ * which is present in order by
+ *
+ * @param orderByDimensions order by dimension
+ * @param generator key generator
+ * @param maskedRanges masked byte range for dimension
+ * @return range of masked byte for order by dimension
+ */
+ public static int[][] getMaskedByteRangeForSorting(List<QueryDimension> orderByDimensions,
+ KeyGenerator generator, int[] maskedRanges) {
+ int[][] dimensionCompareIndex = new int[orderByDimensions.size()][];
+ int index = 0;
+ for (int i = 0; i < dimensionCompareIndex.length; i++) {
+ Set<Integer> integers = new TreeSet<Integer>();
+ if (!orderByDimensions.get(i).getDimension().getEncoder().contains(Encoding.DICTIONARY)
+ || orderByDimensions.get(i).getDimension().numberOfChild() > 0) {
+ continue;
+ }
+ int[] range =
+ generator.getKeyByteOffsets(orderByDimensions.get(i).getDimension().getKeyOrdinal());
+ for (int j = range[0]; j <= range[1]; j++) {
+ integers.add(j);
+ }
+ dimensionCompareIndex[index] = new int[integers.size()];
+ int j = 0;
+ for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
+ Integer integer = (Integer) iterator.next();
+ dimensionCompareIndex[index][j++] = integer.intValue();
+ }
+ index++;
+ }
+ for (int i = 0; i < dimensionCompareIndex.length; i++) {
+ if (null == dimensionCompareIndex[i]) {
+ continue;
+ }
+ int[] range = dimensionCompareIndex[i];
+ if (null != range) {
+ for (int j = 0; j < range.length; j++) {
+ for (int k = 0; k < maskedRanges.length; k++) {
+ if (range[j] == maskedRanges[k]) {
+ range[j] = k;
+ break;
+ }
+ }
+ }
+ }
+
+ }
+ return dimensionCompareIndex;
+ }
+
+ /**
+ * Below method will be used to get the masked key for sorting
+ *
+ * @param orderDimensions query dimension
+ * @param generator key generator
+ * @param maskedByteRangeForSorting masked byte range for sorting
+ * @param maskedRanges masked range
+ * @return masked byte range
+ * @throws QueryExecutionException
+ */
+ public static byte[][] getMaksedKeyForSorting(List<QueryDimension> orderDimensions,
+ KeyGenerator generator, int[][] maskedByteRangeForSorting, int[] maskedRanges)
+ throws QueryExecutionException {
+ byte[][] maskedKey = new byte[orderDimensions.size()][];
+ byte[] mdKey = null;
+ long[] key = null;
+ byte[] maskedMdKey = null;
+ try {
+ if (null != maskedByteRangeForSorting) {
+ for (int i = 0; i < maskedByteRangeForSorting.length; i++) {
+ if (null == maskedByteRangeForSorting[i]) {
+ continue;
+ }
+ key = new long[generator.getDimCount()];
+ maskedKey[i] = new byte[maskedByteRangeForSorting[i].length];
+ key[orderDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE;
+ mdKey = generator.generateKey(key);
+ maskedMdKey = new byte[maskedRanges.length];
+ for (int k = 0; k < maskedMdKey.length; k++) { // CHECKSTYLE:OFF
+ // Approval
+ // No:Approval-V1R2C10_001
+ maskedMdKey[k] = mdKey[maskedRanges[k]];
+ }
+ for (int j = 0; j < maskedByteRangeForSorting[i].length; j++) {
+ maskedKey[i][j] = maskedMdKey[maskedByteRangeForSorting[i][j]];
+ }// CHECKSTYLE:ON
+
+ }
+ }
+ } catch (KeyGenException e) {
+ throw new QueryExecutionException(e);
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method will be used to get mapping whether dimension is present in
+ * order by or not
+ *
+ * @param sortedDimensions sort dimension present in order by query
+ * @param queryDimensions query dimension
+ * @return sort dimension indexes
+ */
+ public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions,
+ List<QueryDimension> queryDimensions) {
+ byte[] sortedDims = new byte[queryDimensions.size()];
+ int indexOf = 0;
+ for (int i = 0; i < sortedDims.length; i++) {
+ indexOf = sortedDimensions.indexOf(queryDimensions.get(i));
+ if (indexOf > -1) {
+ sortedDims[i] = 1;
+ }
+ }
+ return sortedDims;
+ }
+
+ /**
+ * Below method will be used to get the mapping of block index and its
+ * restructuring info
+ *
+ * @param queryDimensions query dimension from query model
+ * @param segmentProperties segment properties
+ * @return map of block index to its restructuring info
+ * @throws KeyGenException if problem while key generation
+ */
+ public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
+ List<QueryDimension> queryDimensions, SegmentProperties segmentProperties)
+ throws KeyGenException {
+ Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
+ // get column group id and its ordinal mapping of column group
+ Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery =
+ getColumnGroupAndItsOrdinalMapping(queryDimensions);
+ Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor =
+ segmentProperties.getColumnGroupAndItsKeygenartor();
+
+ Iterator<Entry<Integer, List<Integer>>> iterator =
+ columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator();
+ KeyStructureInfo restructureInfos = null;
+ while (iterator.hasNext()) {
+ Entry<Integer, List<Integer>> next = iterator.next();
+ KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey());
+ restructureInfos = new KeyStructureInfo();
+ // sort the ordinal
+ List<Integer> ordinal = next.getValue();
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ for (Integer ord : ordinal) {
+ mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
+ }
+ Collections.sort(mdKeyOrdinal);
+ // get the masked byte range for column group
+ int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ // max key for column group
+ byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ // get masked key for column group
+ int[] maksedByte = getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMaskedBytes(maksedByte);
+ rowGroupToItsRSInfo
+ .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
+ restructureInfos);
+ }
+ return rowGroupToItsRSInfo;
+ }
+
+ /**
+ * return true if given key is found in array
+ *
+ * @param data
+ * @param key
+ * @return
+ */
+ public static boolean searchInArray(int[] data, int key) {
+ for (int i = 0; i < data.length; i++) {
+ if (key == data[i]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Below method will be used to create a mapping of column group columns
+ * this mapping will have column group id to all the dimension ordinal
+ * present in the column group This mapping will be used during query
+ * execution, to create a mask key for the column group dimension which will
+ * be used in aggregation and filter query as column group dimension will be
+ * stored in bit level
+ */
+ private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
+ List<QueryDimension> origdimensions) {
+
+ List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size());
+ dimensions.addAll(origdimensions);
+ /**
+ * sort based on column group id
+ */
+ Collections.sort(dimensions, new Comparator<QueryDimension>() {
+
+ @Override public int compare(QueryDimension o1, QueryDimension o2) {
+ return Integer
+ .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
+ }
+ });
+ // list of row groups this will store all the row group column
+ Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping =
+ new HashMap<Integer, List<Integer>>();
+ // to store a column group
+ List<Integer> currentColumnGroup = null;
+ // current index
+ int index = 0;
+ // previous column group to check all the column of row id has bee
+ // selected
+ int prvColumnGroupId = -1;
+ while (index < dimensions.size()) {
+ // if dimension group id is not zero and it is same as the previous
+ // column group id
+ // then we need to add ordinal of that column as it belongs to same
+ // column group
+ if (!dimensions.get(index).getDimension().isColumnar()
+ && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId
+ && null != currentColumnGroup) {
+ currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
+ }
+
+ // if dimension is not a columnar then it is column group column
+ else if (!dimensions.get(index).getDimension().isColumnar()) {
+ currentColumnGroup = new ArrayList<Integer>();
+ columnGroupAndItsOrdinalsMapping
+ .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup);
+ currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
+ }
+ // update the row id every time,this is required to group the
+ // columns
+ // of the same row group
+ prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId();
+ index++;
+ }
+ return columnGroupAndItsOrdinalsMapping;
+ }
+
+ /**
+ * Below method will be used to get masked byte
+ *
+ * @param data actual data
+ * @param maxKey max key
+ * @param maskByteRanges mask byte range
+ * @param byteCount
+ * @return masked byte
+ */
+ public static byte[] getMaskedKey(byte[] data, byte[] maxKey, int[] maskByteRanges,
+ int byteCount) {
+ byte[] maskedKey = new byte[byteCount];
+ int counter = 0;
+ int byteRange = 0;
+ for (int i = 0; i < byteCount; i++) {
+ byteRange = maskByteRanges[i];
+ if (byteRange != -1) {
+ maskedKey[counter++] = (byte) (data[byteRange] & maxKey[byteRange]);
+ }
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method will be used to fill block indexes of the query dimension
+ * which will be used in creating a output row Here is method we are passing
+ * two list which store the indexes one for dictionary column other for not
+ * dictionary column. This is done for specific purpose so that in one
+ * iteration we will be able to fill both type dimension block indexes
+ *
+ * @param queryDimensions dimension present in the query
+ * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping
+ * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes
+ * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes
+ */
+ public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
+ Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
+ Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
+ && queryDimension.getDimension().numberOfChild() == 0) {
+ dictionaryDimensionBlockIndex
+ .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- } else if(queryDimension.getDimension().numberOfChild() == 0){
++ } else if (queryDimension.getDimension().numberOfChild() == 0) {
+ noDictionaryDimensionBlockIndex
+ .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to resolve the query model
+ * resolve will be setting the actual dimension and measure object
+ * as from driver only column name will be passes to avoid the heavy object
+ * serialization
+ *
+ * @param queryModel query model
+ */
+ public static void resolveQueryModel(QueryModel queryModel) {
+ CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
+ // TODO need to load the table from table identifier
+ CarbonTable carbonTable = queryModel.getTable();
+ String tableName =
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
+ // resolve query dimension
+ for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
+ queryDimension
+ .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
+ }
+ // resolve sort dimension
+ for (QueryDimension sortDimension : queryModel.getSortDimension()) {
+ sortDimension
+ .setDimension(carbonTable.getDimensionByName(tableName, sortDimension.getColumnName()));
+ }
+ // resolve query measure
+ for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
+ // in case of count start column name will be count * so
+ // first need to check any measure is present or not and as if measure
+ // if measure is present and if first measure is not a default
+ // measure than add measure otherwise
+ // than add first dimension as a measure
+ //as currently if measure is not present then
+ //we are adding default measure so first condition will
+ //never come false but if in future we can remove so not removing first if check
+ if (queryMeasure.getColumnName().equals("count(*)")) {
+ if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable
+ .getMeasureByTableName(tableName).get(0).getColName()
+ .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
+ queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
+ } else {
+ CarbonMeasure dummyMeasure = new CarbonMeasure(
+ carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
+ queryMeasure.setMeasure(dummyMeasure);
+ }
+ } else {
+ queryMeasure
+ .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to get the index of number type aggregator
+ *
+ * @param aggType
+ * @return index in aggregator
+ */
+ public static int[] getNumberTypeIndex(List<String> aggType) {
+ List<Integer> indexList = new ArrayList<Integer>();
+ for (int i = 0; i < aggType.size(); i++) {
+ if (CarbonCommonConstants.SUM.equals(aggType.get(i)) || CarbonCommonConstants.AVERAGE
+ .equals(aggType.get(i))) {
+ indexList.add(i);
+ }
+ }
+ return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
+ }
+
+ /**
+ * below method will be used to get the actual type aggregator
+ *
+ * @param aggType
+ * @return index in aggrgetor
+ */
+ public static int[] getActualTypeIndex(List<String> aggType) {
+ List<Integer> indexList = new ArrayList<Integer>();
+ for (int i = 0; i < aggType.size(); i++) {
+ if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE
+ .equals(aggType.get(i))) {
+ indexList.add(i);
+ }
+ }
+ return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
+ }
+
+ /**
+ * Below method will be used to get the key structure for the column group
+ *
+ * @param segmentProperties segment properties
+ * @param dimColumnEvaluatorInfo dimension evaluator info
+ * @return key structure info for column group dimension
+ * @throws KeyGenException
+ */
+ public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
+ int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+
+ mdKeyOrdinal.add(segmentProperties
+ .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMaskedBytes(maksedByte);
+ return restructureInfos;
+ }
+
+ /**
+ * Below method will be used to get the column group id based on the ordinal
+ *
+ * @param segmentProperties segment properties
+ * @param ordinal ordinal to be searched
+ * @return column group id
+ */
+ public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ /**
+ * Below method will be used to get the map of for complex dimension and its type
+ * which will be used to during query execution to
+ *
+ * @param queryDimensions complex dimension in query
+ * @param dimensionToBlockIndexMap dimension to block index in file map
+ * @return complex dimension and query type
+ */
+ public static Map<Integer, GenericQueryType> getComplexDimensionsMap(
+ List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
+ int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap,
+ Set<CarbonDimension> filterDimensions) {
+ Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>();
+ for (QueryDimension dimension : queryDimensions) {
+ CarbonDimension actualDimension = dimension.getDimension();
+ if (actualDimension.getNumberOfChild() == 0) {
+ continue;
+ }
+ fillParentDetails(dimensionToBlockIndexMap, actualDimension, complexTypeMap,
+ eachComplexColumnValueSize, columnIdToDictionaryMap);
+ }
+ if (null != filterDimensions) {
+ for (CarbonDimension filterDimension : filterDimensions) {
+ fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
+ eachComplexColumnValueSize, columnIdToDictionaryMap);
+ }
+ }
+ return complexTypeMap;
+ }
+
+ private static GenericQueryType fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
+ CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap,
+ int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
+ int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
+ GenericQueryType parentQueryType = dimension.getDataType().equals(DataType.ARRAY) ?
+ new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex) :
+ new StructQueryType(dimension.getColName(), dimension.getColName(),
+ dimensionToBlockIndexMap.get(dimension.getOrdinal()));
+ complexTypeMap.put(dimension.getOrdinal(), parentQueryType);
+ parentBlockIndex =
+ fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
+ dimension, parentQueryType);
+ return parentQueryType;
+ }
+
+ private static int fillChildrenDetails(int[] eachComplexColumnValueSize,
+ Map<String, Dictionary> columnIdToDictionaryMap, int parentBlockIndex,
+ CarbonDimension dimension, GenericQueryType parentQueryType) {
+ for (int i = 0; i < dimension.getNumberOfChild(); i++) {
+ switch (dimension.getListOfChildDimensions().get(i).getDataType()) {
+ case ARRAY:
+ parentQueryType.addChildren(
+ new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex));
+ break;
+ case STRUCT:
+ parentQueryType.addChildren(
+ new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex));
+ break;
+ default:
- boolean isDirectDictionary = CarbonUtil.hasEncoding(
- dimension.getListOfChildDimensions().get(i).getEncoder(),
- Encoding.DIRECT_DICTIONARY);
++ boolean isDirectDictionary = CarbonUtil
++ .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
++ Encoding.DIRECT_DICTIONARY);
+ parentQueryType.addChildren(
+ new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex,
+ dimension.getListOfChildDimensions().get(i).getDataType(),
+ eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i)
+ .getComplexTypeOrdinal()], columnIdToDictionaryMap
+ .get(dimension.getListOfChildDimensions().get(i).getColumnId()),
+ isDirectDictionary));
+ }
+ if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
+ parentBlockIndex = fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap,
+ parentBlockIndex, dimension.getListOfChildDimensions().get(i), parentQueryType);
+ }
+ }
+ return parentBlockIndex;
+ }
+
+ public static Set<CarbonDimension> getAllFilterDimensions(FilterResolverIntf filterResolverTree) {
+ Set<CarbonDimension> filterDimensions = new HashSet<CarbonDimension>();
+ if (null == filterResolverTree) {
+ return filterDimensions;
+ }
+ List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>();
+ Expression filterExpression = filterResolverTree.getFilterExpression();
- if (filterExpression instanceof BinaryLogicalExpression) {
- BinaryLogicalExpression logicalExpression = (BinaryLogicalExpression) filterExpression;
- dimensionResolvedInfos.addAll(logicalExpression.getColumnList());
- }
++ addColumnDimensions(filterExpression, filterDimensions);
+ for (ColumnExpression info : dimensionResolvedInfos) {
+ if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
+ filterDimensions.add(info.getDimension());
+ }
+ }
+ return filterDimensions;
+
+ }
+
++ /**
++ * This method will check if a given expression contains a column expression
++ * recursively and add the dimension instance to the set which holds the dimension
++ * instances of the complex filter expressions.
++ *
++ * @param filterDimensions
++ * @return
++ */
++ private static void addColumnDimensions(Expression expression,
++ Set<CarbonDimension> filterDimensions) {
++ if (null != expression && expression instanceof ColumnExpression
++ && ((ColumnExpression) expression).isDimension()) {
++ filterDimensions.add(((ColumnExpression) expression).getDimension());
++ return;
++ }
++ for (Expression child : expression.getChildren()) {
++ addColumnDimensions(child, filterDimensions);
++ }
++ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/50dfdf6c/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
index c58331d,0000000..a2c6f28
mode 100644,000000..100644
--- a/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/carbondata/scan/filter/FilterExpressionProcessor.java
@@@ -1,350 -1,0 +1,353 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.scan.filter;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.datastore.DataRefNode;
+import org.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.carbondata.core.carbon.datastore.IndexKey;
+import org.carbondata.core.carbon.datastore.block.AbstractIndex;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.carbondata.core.carbon.metadata.datatype.DataType;
+import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.scan.executor.exception.QueryExecutionException;
+import org.carbondata.scan.expression.BinaryExpression;
+import org.carbondata.scan.expression.Expression;
+import org.carbondata.scan.expression.conditional.BinaryConditionalExpression;
+import org.carbondata.scan.expression.conditional.ConditionalExpression;
+import org.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.carbondata.scan.expression.logical.BinaryLogicalExpression;
+import org.carbondata.scan.filter.executer.FilterExecuter;
+import org.carbondata.scan.filter.intf.ExpressionType;
+import org.carbondata.scan.filter.resolver.ConditionalFilterResolverImpl;
+import org.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.carbondata.scan.filter.resolver.LogicalFilterResolverImpl;
+import org.carbondata.scan.filter.resolver.RowLevelFilterResolverImpl;
+import org.carbondata.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
+
+public class FilterExpressionProcessor implements FilterProcessor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(FilterExpressionProcessor.class.getName());
+
+ /**
+ * Implementation will provide the resolved form of filters based on the
+ * filter expression tree which is been passed in Expression instance.
+ *
+ * @param expressionTree , filter expression tree
+ * @param tableIdentifier ,contains carbon store informations
+ * @return a filter resolver tree
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ public FilterResolverIntf getFilterResolver(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ if (null != expressionTree && null != tableIdentifier) {
+ return getFilterResolvertree(expressionTree, tableIdentifier);
+ }
+ return null;
+ }
+
+ /**
+ * This API will scan the Segment level all btrees and selects the required
+ * block reference nodes inorder to push the same to executer for applying filters
+ * on the respective data reference node.
+ * Following Algorithm is followed in below API
+ * Step:1 Get the start end key based on the filter tree resolver information
+ * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
+ * node(block)
+ * Step:3 Once data reference node ranges retrieved traverse the node within this range
+ * and select the node based on the block min and max value and the filter value.
+ * Step:4 The selected blocks will be send to executers for applying the filters with the help
+ * of Filter executers.
+ *
+ * @throws QueryExecutionException
+ */
+ public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
+ FilterResolverIntf filterResolver, AbstractIndex tableSegment,
+ AbsoluteTableIdentifier tableIdentifier) throws QueryExecutionException {
+ // Need to get the current dimension tables
+ List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
+ // getting the start and end index key based on filter for hitting the
+ // selected block reference nodes based on filter resolver tree.
- LOGGER.info("preparing the start and end key for finding"
++ LOGGER.debug("preparing the start and end key for finding"
+ + "start and end block as per filter resolver");
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
+ tableIdentifier, filterResolver, listOfStartEndKeys);
+ // reading the first value from list which has start key
+ IndexKey searchStartKey = listOfStartEndKeys.get(0);
+ // reading the last value from list which has end key
+ IndexKey searchEndKey = listOfStartEndKeys.get(1);
+ if (null == searchStartKey && null == searchEndKey) {
+ try {
+ // TODO need to handle for no dictionary dimensions
+ searchStartKey =
+ FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
+ // TODO need to handle for no dictionary dimensions
+ searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
+ } catch (KeyGenException e) {
+ return listOfDataBlocksToScan;
+ }
+ }
+
- LOGGER.info("Successfully retrieved the start and end key");
++ LOGGER.debug(
++ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
++ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
++ + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
++ + searchEndKey.getNoDictionaryKeys());
+ long startTimeInMillis = System.currentTimeMillis();
+ DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
+ DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(),null);
+ while (startBlock != endBlock) {
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock,
+ tableSegment.getSegmentProperties());
+ startBlock = startBlock.getNextDataRefNode();
+ }
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock,
+ tableSegment.getSegmentProperties());
+ LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (
+ System.currentTimeMillis() - startTimeInMillis)
+ + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan
+ .size());
+
+ return listOfDataBlocksToScan;
+ }
+
+ /**
+ * Selects the blocks based on col max and min value.
+ *
+ * @param filterResolver
+ * @param listOfDataBlocksToScan
+ * @param dataRefNode
+ * @param segmentProperties
+ */
+ private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter,
+ List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode,
+ SegmentProperties segmentProperties) {
+
+ BitSet bitSet = filterExecuter
+ .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue());
+ if (!bitSet.isEmpty()) {
+ listOfDataBlocksToScan.add(dataRefNode);
+
+ }
+ }
+
+ /**
+ * API will return a filter resolver instance which will be used by
+ * executers to evaluate or execute the filters.
+ *
+ * @param expressionTree , resolver tree which will hold the resolver tree based on
+ * filter expression.
+ * @return FilterResolverIntf type.
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ private FilterResolverIntf getFilterResolvertree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ FilterResolverIntf filterEvaluatorTree =
+ createFilterResolverTree(expressionTree, tableIdentifier, null);
+ traverseAndResolveTree(filterEvaluatorTree, tableIdentifier);
+ return filterEvaluatorTree;
+ }
+
+ /**
+ * constructing the filter resolver tree based on filter expression.
+ * this method will visit each node of the filter resolver and prepares
+ * the surrogates of the filter members which are involved filter
+ * expression.
+ *
+ * @param filterResolverTree
+ * @param tableIdentifier
+ * @throws FilterUnsupportedException
+ * @throws QueryExecutionException
+ */
+ private void traverseAndResolveTree(FilterResolverIntf filterResolverTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ if (null == filterResolverTree) {
+ return;
+ }
+ traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier);
+
+ filterResolverTree.resolve(tableIdentifier);
+
+ traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier);
+ }
+
+ /**
+ * Pattern used : Visitor Pattern
+ * Method will create filter resolver tree based on the filter expression tree,
+ * in this algorithm based on the expression instance the resolvers will created
+ *
+ * @param expressionTree
+ * @param tableIdentifier
+ * @return
+ */
+ private FilterResolverIntf createFilterResolverTree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier, Expression intermediateExpression) {
+ ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
+ BinaryExpression currentExpression = null;
- BinaryLogicalExpression logicalExpression = null;
+ switch (filterExpressionType) {
+ case OR:
+ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
+ currentExpression),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
- currentExpression), filterExpressionType);
++ currentExpression),currentExpression);
+ case AND:
- logicalExpression = (BinaryLogicalExpression) expressionTree;
++ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
- createFilterResolverTree(logicalExpression.getLeft(), tableIdentifier,
++ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
+ currentExpression),
- createFilterResolverTree(logicalExpression.getRight(), tableIdentifier,
- currentExpression), filterExpressionType);
++ createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
++ currentExpression), currentExpression);
+ case EQUALS:
+ case IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, false, expressionTree,
+ tableIdentifier, expressionTree);
+ case GREATERTHAN:
+ case GREATERTHAN_EQUALTO:
+ case LESSTHAN:
+ case LESSTHAN_EQUALTO:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, true, expressionTree,
+ tableIdentifier, expressionTree);
+
+ case NOT_EQUALS:
+ case NOT_IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.NOT_EQUALS, false,
+ expressionTree, tableIdentifier, expressionTree);
+
+ default:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.UNKNOWN, false, expressionTree,
+ tableIdentifier, expressionTree);
+ }
+ }
+
+ /**
+ * Factory method which will return the resolver instance based on filter expression
+ * expressions.
+ */
+ private FilterResolverIntf getFilterResolverBasedOnExpressionType(
+ ExpressionType filterExpressionType, boolean isExpressionResolve, Expression expression,
+ AbsoluteTableIdentifier tableIdentifier, Expression expressionTree) {
+ BinaryConditionalExpression currentCondExpression = null;
+ ConditionalExpression condExpression = null;
+ switch (filterExpressionType) {
+ case EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ // getting new dim index.
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.GREATERTHAN_EQUALTO
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.LESSTHAN_EQUALTO) {
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true);
+
+ }
+ break;
+ case NOT_EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+ if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
+
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ break;
+ default:
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.isSingleDimension()
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new ConditionalFilterResolverImpl(expression, true, true);
+ } else {
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+ } else {
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+ }
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+
+}
[27/47] incubator-carbondata git commit: [CARBONDATA-115] The log
level has been changed to debug since its been repeatedly getting logged for
each block, so if any detail level analysis is required (#863)
Posted by ra...@apache.org.
[CARBONDATA-115] The log level has been changed to debug since its been repeatedly getting logged for each block, so if any detail level analysis is required (#863)
the developer can verify the debug level log to know how much time is taken to generate start and end key.
Also as part of log start and end key for the block will be logged.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6dfaefed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6dfaefed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6dfaefed
Branch: refs/heads/master
Commit: 6dfaefed8d0de5245d7fef72573a555134ae4d00
Parents: 14a46b2
Author: sujith71955 <su...@gmail.com>
Authored: Thu Jul 28 15:47:53 2016 +0530
Committer: Kumar Vishal <ku...@gmail.com>
Committed: Thu Jul 28 15:47:53 2016 +0530
----------------------------------------------------------------------
.../carbondata/query/filters/FilterExpressionProcessor.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6dfaefed/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
index 9482efe..e25b051 100644
--- a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
@@ -94,7 +94,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
// getting the start and end index key based on filter for hitting the
// selected block reference nodes based on filter resolver tree.
- LOGGER.info("preparing the start and end key for finding"
+ LOGGER.debug("preparing the start and end key for finding"
+ "start and end block as per filter resolver");
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
@@ -115,7 +115,11 @@ public class FilterExpressionProcessor implements FilterProcessor {
}
}
- LOGGER.info("Successfully retrieved the start and end key");
+ LOGGER.debug(
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
+ + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
+ + searchEndKey.getNoDictionaryKeys());
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
tableSegment.getSegmentProperties().getEachDimColumnValueSize());
[15/47] incubator-carbondata git commit: [bug]fix store format bug in
complex type example(#477)
Posted by ra...@apache.org.
[bug]fix store format bug in complex type example(#477)
[bug]fix store format bug in complex type example(#477)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9f269d19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9f269d19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9f269d19
Branch: refs/heads/master
Commit: 9f269d196900f3a233b877488e5c58a0ac730846
Parents: 1b16f76
Author: Gin-zhj <zh...@huawei.com>
Authored: Mon Jul 25 11:52:04 2016 +0800
Committer: david <qi...@qq.com>
Committed: Mon Jul 25 11:52:04 2016 +0800
----------------------------------------------------------------------
.../scala/org/carbondata/examples/ComplexTypeExample.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9f269d19/examples/src/main/scala/org/carbondata/examples/ComplexTypeExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/carbondata/examples/ComplexTypeExample.scala b/examples/src/main/scala/org/carbondata/examples/ComplexTypeExample.scala
index 070dff1..2202a9c 100644
--- a/examples/src/main/scala/org/carbondata/examples/ComplexTypeExample.scala
+++ b/examples/src/main/scala/org/carbondata/examples/ComplexTypeExample.scala
@@ -55,10 +55,10 @@ object ComplexTypeExample {
activeDeactivedate: array<string>>,
gamePointId double,
contractNumber double)
- row format delimited fields terminated by ','
- collection items terminated by '$$' map keys terminated by ':' """)
+ STORED BY 'org.apache.carbondata.format' """)
- cc.sql(s"LOAD DATA LOCAL INPATH '$dataPath' INTO TABLE $tableName")
+ cc.sql(s"load data local inpath '$dataPath' into table $tableName " +
+ "options ('COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')")
// filter on complex ARRAY type with index filter
cc.sql(s"SELECT mobile, proddate.activeDeactivedate, MAC[0] FROM $tableName " +
[11/47] incubator-carbondata git commit: [CARBONDATA-96] Make
zookeeper lock as default if zookeeper url is configured. make zookeeper as
default if zookeeper url is configured. (#712)
Posted by ra...@apache.org.
[CARBONDATA-96] Make zookeeper lock as default if zookeeper url is configured. make zookeeper as default if zookeeper url is configured. (#712)
Make the lock type as zookeeper if zookeeper URL is present in the spark conf.
if spark.deploy.zookeeper.url property is set in spark-default.conf then need to take the zookeeper locking.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c3197a86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c3197a86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c3197a86
Branch: refs/heads/master
Commit: c3197a8645b5b2d06b298b07888693cfa1c2e793
Parents: 30e4f25
Author: ravikiran23 <ra...@gmail.com>
Authored: Sat Jul 23 12:55:12 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jul 23 12:55:12 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/core/locks/CarbonLockFactory.java | 5 ++---
.../apache/spark/sql/hive/CarbonMetastoreCatalog.scala | 11 +++++++----
2 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c3197a86/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
index 8616c68..b650cd4 100644
--- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
@@ -34,7 +34,7 @@ public class CarbonLockFactory {
private static String lockTypeConfigured;
static {
- CarbonLockFactory.updateZooKeeperLockingStatus();
+ CarbonLockFactory.getLockTypeConfigured();
}
/**
@@ -86,10 +86,9 @@ public class CarbonLockFactory {
/**
* This method will set the zookeeper status whether zookeeper to be used for locking or not.
*/
- private static void updateZooKeeperLockingStatus() {
+ private static void getLockTypeConfigured() {
lockTypeConfigured = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c3197a86/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index f952aa3..5966240 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -207,11 +207,14 @@ class CarbonMetastoreCatalog(hive: HiveContext, val storePath: String, client: C
// creating zookeeper instance once.
// if zookeeper is configured as carbon lock type.
- if (CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT)
- .equalsIgnoreCase(CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER)) {
- val zookeeperUrl = hive.getConf("spark.deploy.zookeeper.url", "127.0.0.1:2181")
+ val zookeeperUrl: String = hive.getConf("spark.deploy.zookeeper.url", null)
+ if (zookeeperUrl != null) {
ZookeeperInit.getInstance(zookeeperUrl)
+ LOGGER.info("Zookeeper url is configured. Taking the zookeeper as lock type.")
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.LOCK_TYPE,
+ CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER
+ )
}
if (metadataPath == null) {