You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/01 15:29:43 UTC
[1/2] incubator-carbondata git commit: Insert into carbon table new
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 997af85dc -> 90bc36699
Insert into carbon table new
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5f1abef7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5f1abef7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5f1abef7
Branch: refs/heads/master
Commit: 5f1abef794d22110f2b40fc0fabad19c7f215f0b
Parents: 997af85
Author: ashok.blend <as...@gmail.com>
Authored: Mon Nov 28 21:20:45 2016 -0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 1 23:28:40 2016 +0800
----------------------------------------------------------------------
.../ThriftWrapperSchemaConverterImpl.java | 2 +
.../metadata/schema/table/CarbonTable.java | 80 ++++++++-
.../schema/table/column/CarbonColumn.java | 14 +-
.../schema/table/column/CarbonDimension.java | 7 +-
.../schema/table/column/CarbonMeasure.java | 6 +-
.../schema/table/column/ColumnSchema.java | 8 +
.../core/constants/CarbonCommonConstants.java | 9 +
.../carbondata/scan/filter/FilterUtilTest.java | 8 +-
format/src/main/thrift/schema.thrift | 7 +-
.../execution/command/carbonTableSchema.scala | 15 +-
.../spark/sql/CarbonCatalystOperators.scala | 21 +++
.../org/apache/spark/sql/CarbonContext.scala | 2 +-
.../spark/sql/CarbonDatasourceRelation.scala | 43 ++++-
.../scala/org/apache/spark/sql/CarbonScan.scala | 43 ++---
.../org/apache/spark/sql/CarbonSqlParser.scala | 31 ++--
.../execution/command/carbonTableSchema.scala | 22 +++
.../sql/hive/CarbonPreInsertionCasts.scala | 59 ++++++
.../spark/sql/hive/CarbonStrategies.scala | 4 +
.../spark/sql/optimizer/CarbonOptimizer.scala | 1 +
.../spark/src/test/resources/shortolap.csv | 5 +
.../TestLoadDataWithSingleQuotechar.scala | 8 +-
.../InsertIntoCarbonTableTestCase.scala | 178 +++++++++++++++++++
.../TestNoInvertedIndexLoadAndQuery.scala | 2 +-
23 files changed, 500 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 8a53895..c1020e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -167,6 +167,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.setColumnProperties(wrapperColumnSchema.getColumnProperties());
thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
+ thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
return thriftColumnSchema;
}
@@ -339,6 +340,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setColumnProperties(externalColumnSchema.getColumnProperties());
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
+ wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index c0a7032..d3e2e62 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -21,7 +21,10 @@ package org.apache.carbondata.core.carbon.metadata.schema.table;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -30,6 +33,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
@@ -61,6 +65,7 @@ public class CarbonTable implements Serializable {
*/
private Map<String, List<CarbonDimension>> tableDimensionsMap;
+ private Map<String, List<CarbonColumn>> createOrderColumn;
/**
* table measures list.
*/
@@ -95,6 +100,7 @@ public class CarbonTable implements Serializable {
this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
this.aggregateTablesName = new ArrayList<String>();
+ this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
}
/**
@@ -113,6 +119,7 @@ public class CarbonTable implements Serializable {
new AbsoluteTableIdentifier(tableInfo.getStorePath(), carbontableIdentifier);
fillDimensionsAndMeasuresForTables(tableInfo.getFactTable());
+ fillCreateOrderColumn(tableInfo.getFactTable().getTableName());
List<TableSchema> aggregateTableList = tableInfo.getAggregateTableList();
for (TableSchema aggTable : aggregateTableList) {
this.aggregateTablesName.add(aggTable.getTableName());
@@ -121,6 +128,33 @@ public class CarbonTable implements Serializable {
}
/**
+ * fill columns as per user provided order
+ * @param tableName
+ */
+ private void fillCreateOrderColumn(String tableName) {
+ List<CarbonColumn> columns = new ArrayList<CarbonColumn>();
+ List<CarbonDimension> dimensions = this.tableDimensionsMap.get(tableName);
+ List<CarbonMeasure> measures = this.tableMeasuresMap.get(tableName);
+ Iterator<CarbonDimension> dimItr = dimensions.iterator();
+ while (dimItr.hasNext()) {
+ columns.add(dimItr.next());
+ }
+ Iterator<CarbonMeasure> msrItr = measures.iterator();
+ while (msrItr.hasNext()) {
+ columns.add(msrItr.next());
+ }
+ Collections.sort(columns, new Comparator<CarbonColumn>() {
+
+ @Override public int compare(CarbonColumn o1, CarbonColumn o2) {
+
+ return Integer.compare(o1.getSchemaOrdinal(), o2.getSchemaOrdinal());
+ }
+
+ });
+ this.createOrderColumn.put(tableName, columns);
+ }
+
+ /**
* This method will return the table size. Default table block size will be considered
* in case not specified by the user
*
@@ -166,7 +200,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, ++complexTypeOrdinal);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
dimensions.add(complexDimension);
dimensionOrdinal =
@@ -176,22 +211,26 @@ public class CarbonTable implements Serializable {
complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
- dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
+ dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, -1));
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
&& columnSchema.getColumnGroupId() == -1) {
dimensions
- .add(new CarbonDimension(columnSchema, dimensionOrdinal++, keyOrdinal++, -1, -1));
+ .add(new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1));
} else {
columnGroupOrdinal =
previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0;
previousColumnGroupId = columnSchema.getColumnGroupId();
- dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++, keyOrdinal++,
+ dimensions.add(new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), keyOrdinal++,
columnGroupOrdinal, -1));
}
}
} else {
- measures.add(new CarbonMeasure(columnSchema, measureOrdinal++));
+ measures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
+ columnSchema.getSchemaOrdinal()));
}
}
}
@@ -213,7 +252,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -221,7 +261,8 @@ public class CarbonTable implements Serializable {
listOfColumns, complexDimension);
} else {
parentDimension.getListOfChildDimensions()
- .add(new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, -1));
+ .add(new CarbonDimension(columnSchema, dimensionOrdinal++,
+ columnSchema.getSchemaOrdinal(), -1, -1, -1));
}
}
}
@@ -336,6 +377,15 @@ public class CarbonTable implements Serializable {
}
/**
+ * This will give user created order column
+ *
+ * @return
+ */
+ public List<CarbonColumn> getCreateOrderColumn(String tableName) {
+ return createOrderColumn.get(tableName);
+ }
+
+ /**
* to get particular measure from a table
*
* @param tableName
@@ -370,6 +420,22 @@ public class CarbonTable implements Serializable {
}
/**
+ * @param tableName
+ * @param columnName
+ * @return
+ */
+ public CarbonColumn getColumnByName(String tableName, String columnName) {
+ List<CarbonColumn> columns = createOrderColumn.get(tableName);
+ Iterator<CarbonColumn> colItr = columns.iterator();
+ while (colItr.hasNext()) {
+ CarbonColumn col = colItr.next();
+ if (col.getColName().equalsIgnoreCase(columnName)) {
+ return col;
+ }
+ }
+ return null;
+ }
+ /**
* gets all children dimension for complex type
*
* @param dimName
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java
index 2cef374..9cf3601 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java
@@ -45,6 +45,10 @@ public class CarbonColumn implements Serializable {
protected int ordinal;
/**
+ * order in which user has created table
+ */
+ protected int schemaOrdinal;
+ /**
* default value for in case of restructuring will be used when older
* segment does not have particular column
*/
@@ -55,13 +59,13 @@ public class CarbonColumn implements Serializable {
*/
protected ColumnIdentifier columnIdentifier;
- public CarbonColumn(ColumnSchema columnSchema, int ordinal) {
+ public CarbonColumn(ColumnSchema columnSchema, int ordinal, int schemaOrdinal) {
this.columnSchema = columnSchema;
this.ordinal = ordinal;
+ this.schemaOrdinal = schemaOrdinal;
this.columnIdentifier =
- new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType());
+ new ColumnIdentifier(getColumnId(), getColumnProperties(), getDataType());
}
-
/**
* @return columnar or row based
*/
@@ -171,4 +175,8 @@ public class CarbonColumn implements Serializable {
public ColumnIdentifier getColumnIdentifier() {
return this.columnIdentifier;
}
+
+ public int getSchemaOrdinal() {
+ return this.schemaOrdinal;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
index c81cea4..214d990 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonDimension.java
@@ -54,8 +54,13 @@ public class CarbonDimension extends CarbonColumn {
private int complexTypeOrdinal;
public CarbonDimension(ColumnSchema columnSchema, int ordinal, int keyOrdinal,
+ int columnGroupOrdinal, int complexTypeOrdinal) {
+ this(columnSchema, ordinal, 0, keyOrdinal, columnGroupOrdinal, complexTypeOrdinal);
+ }
+
+ public CarbonDimension(ColumnSchema columnSchema, int ordinal, int schemaOrdinal, int keyOrdinal,
int columnGroupOrdinal, int complexTypeOrdinal) {
- super(columnSchema, ordinal);
+ super(columnSchema, ordinal, schemaOrdinal);
this.keyOrdinal = keyOrdinal;
this.columnGroupOrdinal = columnGroupOrdinal;
this.complexTypeOrdinal = complexTypeOrdinal;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
index 2c8c11e..ae0a25a 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonMeasure.java
@@ -45,7 +45,11 @@ public class CarbonMeasure extends CarbonColumn {
private int precision;
public CarbonMeasure(ColumnSchema columnSchema, int ordinal) {
- super(columnSchema, ordinal);
+ this(columnSchema, ordinal, 0);
+ }
+
+ public CarbonMeasure(ColumnSchema columnSchema, int ordinal, int schemaOrdinal) {
+ super(columnSchema, ordinal, schemaOrdinal);
this.scale = columnSchema.getScale();
this.precision = columnSchema.getPrecision();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
index 0431ad5..c0b2569 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java
@@ -92,6 +92,7 @@ public class ColumnSchema implements Serializable {
private int precision;
+ private int schemaOrdinal;
/**
* Nested fields. Since thrift does not support nested fields,
* the nesting is flattened to a single list by a depth-first traversal.
@@ -415,4 +416,11 @@ public class ColumnSchema implements Serializable {
this.columnReferenceId = columnReferenceId;
}
+ public int getSchemaOrdinal() {
+ return schemaOrdinal;
+ }
+
+ public void setSchemaOrdinal(int schemaOrdinal) {
+ this.schemaOrdinal = schemaOrdinal;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index a9e0442..443c8c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -910,6 +910,15 @@ public final class CarbonCommonConstants {
* data file version header
*/
public static final String CARBON_DATA_VERSION_HEADER = "CARBONDATAVERSION#";
+ /**
+ * Maximum no of column supported
+ */
+ public static int DEFAULT_MAX_NUMBER_OF_COLUMNS = 20000;
+
+ /**
+ * maximum length of column
+ */
+ public static final int DEFAULT_COLUMN_LENGTH = 100000;
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
index ba4c46a..23e2d10 100644
--- a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
@@ -264,7 +264,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
dimColumn.setDimensionColumn(true);
- final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal);
+ final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
new MockUp<ColumnExpression>() {
@Mock public CarbonColumn getCarbonColumn() {
return carbonColumn;
@@ -295,7 +295,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
dimColumn.setDataType(DataType.STRING);
dimColumn.setDimensionColumn(true);
- final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal);
+ final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
new MockUp<ColumnExpression>() {
@Mock public CarbonColumn getCarbonColumn() {
return carbonColumn;
@@ -318,7 +318,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
int defaultSurrogate = 1;
int ordinal = 1;
- final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal);
+ final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
new MockUp<ColumnExpression>() {
@Mock public CarbonColumn getCarbonColumn() {
return carbonColumn;
@@ -385,7 +385,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
}
};
- final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal);
+ final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
new MockUp<ColumnExpression>() {
@Mock public CarbonColumn getCarbonColumn() {
return carbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 33bb14e..885beba 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -97,8 +97,11 @@ struct ColumnSchema{
/**
* column reference id
*/
- 15: optional string columnReferenceId;
-
+ 15: optional string columnReferenceId;
+ /**
+ * It will have column order which user has provided
+ */
+ 16: optional i32 schemaOrdinal
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 9360ad8..7416c90 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -55,6 +55,7 @@ case class TableModel(
case class Field(column: String, var dataType: Option[String], name: Option[String],
children: Option[List[Field]], parent: String = null,
storeType: Option[String] = Some("columnar"),
+ var schemaOrdinal: Int = -1,
var precision: Int = 0, var scale: Int = 0)
case class ColumnProperty(key: String, value: String)
@@ -126,7 +127,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
val columnSchema: ColumnSchema = getColumnSchema(
DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")),
field.name.getOrElse(field.column), index,
- isCol = true, encoders, isDimensionCol = true, rowGroup, field.precision, field.scale)
+ isCol = true, encoders, isDimensionCol = true, rowGroup, field.precision, field.scale,
+ field.schemaOrdinal)
allColumns ++= Seq(columnSchema)
index = index + 1
rowGroup = rowGroup + 1
@@ -141,7 +143,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
def getColumnSchema(dataType: DataType, colName: String, index: Integer, isCol: Boolean,
encoders: java.util.List[Encoding], isDimensionCol: Boolean,
- colGroup: Integer, precision: Integer, scale: Integer): ColumnSchema = {
+ colGroup: Integer, precision: Integer, scale: Integer, schemaOrdinal: Int): ColumnSchema = {
val columnSchema = new ColumnSchema()
columnSchema.setDataType(dataType)
columnSchema.setColumnName(colName)
@@ -169,6 +171,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
columnSchema.setColumnGroup(colGroup)
columnSchema.setPrecision(precision)
columnSchema.setScale(scale)
+ columnSchema.setSchemaOrdinal(schemaOrdinal)
// TODO: Need to fill RowGroupID, converted type
// & Number of Children after DDL finalization
columnSchema
@@ -191,7 +194,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
isDimensionCol = true,
-1,
field.precision,
- field.scale)
+ field.scale,
+ field.schemaOrdinal)
allColumns ++= Seq(columnSchema)
index = index + 1
if (field.children.isDefined && field.children.get != null) {
@@ -211,7 +215,8 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
isDimensionCol = false,
-1,
field.precision,
- field.scale)
+ field.scale,
+ field.schemaOrdinal)
val measureCol = columnSchema
allColumns ++= Seq(measureCol)
@@ -274,7 +279,7 @@ class TableNewProcessor(cm: TableModel, sqlContext: SQLContext) {
true,
encoders,
false,
- -1, 0, 0)
+ -1, 0, 0, schemaOrdinal = -1)
columnSchema.setInvisible(true)
val measureColumn = columnSchema
measures += measureColumn
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
index d394bfa..01904ba 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonCatalystOperators.scala
@@ -105,3 +105,24 @@ case class DropDatabase(dbName: String, isCascade: Boolean, sql: String)
Seq()
}
}
+
+/**
+ * A logical plan representing insertion into Hive table.
+ * This plan ignores nullability of ArrayType, MapType, StructType unlike InsertIntoTable
+ * because Hive table doesn't have nullability for ARRAY, MAP, STRUCT types.
+ */
+case class InsertIntoCarbonTable(
+ table: CarbonDatasourceRelation,
+ partition: Map[String, Option[String]],
+ child: LogicalPlan,
+ overwrite: Boolean,
+ ifNotExists: Boolean)
+ extends LogicalPlan with Command {
+
+ override def children: Seq[LogicalPlan] = child :: Nil
+ override def output: Seq[Attribute] = Seq.empty
+
+ // This is the expected schema of the table prepared to be inserted into,
+ // including dynamic partition columns.
+ val tableOutput = table.carbonRelation.output
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
index de43319..e5d3d6b 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
@@ -78,7 +78,7 @@ class CarbonContext(
override val extendedResolutionRules =
catalog.ParquetConversions ::
catalog.CreateTables ::
- catalog.PreInsertionCasts ::
+ CarbonPreInsertionCasts ::
ExtractPythonUDFs ::
ResolveHiveWindowFunction ::
PreInsertCastAndRename ::
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 681c0c8..a534ab0 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import java.util.LinkedHashSet
import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
import scala.language.implicitConversions
import org.apache.hadoop.fs.Path
@@ -32,7 +33,7 @@ import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes}
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{DataType, StructType}
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.carbon.path.CarbonStorePath
import org.apache.carbondata.core.datastorage.store.impl.FileFactory
import org.apache.carbondata.lcm.status.SegmentStatusManager
@@ -154,7 +155,7 @@ private[sql] case class CarbonDatasourceRelation(
case class CarbonRelation(
databaseName: String,
tableName: String,
- metaData: CarbonMetaData,
+ var metaData: CarbonMetaData,
tableMeta: TableMeta,
alias: Option[String])(@transient sqlContext: SQLContext)
extends LeafNode with MultiInstanceRelation {
@@ -245,8 +246,40 @@ case class CarbonRelation(
nullable = true)(qualifiers = tableName +: alias.toSeq))
}
- override val output = dimensionsAttr ++ measureAttr
-
+ override val output = {
+ val factTable = tableMeta.carbonTable.getFactTableName
+ var columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
+ .asScala
+ columns.filter(!_.getColumnSchema.isInvisible).map { column =>
+ if (column.isDimesion()) {
+ val output: DataType = column.getDataType.toString.toLowerCase match {
+ case "array" =>
+ CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>")
+ case "struct" =>
+ CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>")
+ case dType =>
+ val dataType = addDecimalScaleAndPrecision(column, dType)
+ CarbonMetastoreTypes.toDataType(dataType)
+ }
+ AttributeReference(column.getColName, output,
+ nullable = true
+ )(qualifiers = tableName +: alias.toSeq)
+ } else {
+ AttributeReference(column.getColName, CarbonMetastoreTypes.toDataType(
+ column.getDataType.toString
+ .toLowerCase match {
+ case "int" => "long"
+ case "short" => "long"
+ case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
+ .getColumnSchema.getScale + ")"
+ case others => others
+ }
+ ),
+ nullable = true
+ )(qualifiers = tableName +: alias.toSeq)
+ }
+ }
+ }
// TODO: Use data from the footers.
override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
@@ -258,7 +291,7 @@ case class CarbonRelation(
}
}
- def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
+ def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
var dType = dataType
if (dimval.getDataType
== org.apache.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 848e752..6f3b5df 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -88,39 +88,28 @@ case class CarbonScan(
columnProjection = attributeOut
}
- val dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName)
- val measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName)
- val dimAttr = new Array[Attribute](dimensions.size())
- val msrAttr = new Array[Attribute](measures.size())
+ val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
+ val colAttr = new Array[Attribute](columns.size())
columnProjection.foreach { attr =>
- val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if(carbonDimension != null) {
- dimAttr(dimensions.indexOf(carbonDimension)) = attr
- } else {
- val carbonMeasure =
- carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
- if(carbonMeasure != null) {
- msrAttr(measures.indexOf(carbonMeasure)) = attr
- }
- }
+ val column =
+ carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
+ if(column != null) {
+ colAttr(columns.indexOf(column)) = attr
+ }
}
- columnProjection = dimAttr.filter(f => f != null) ++ msrAttr.filter(f => f != null)
+ columnProjection = colAttr.filter(f => f != null)
var queryOrder: Integer = 0
columnProjection.foreach { attr =>
- val carbonDimension =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, attr.name)
- if (carbonDimension != null) {
- val dim = new QueryDimension(attr.name)
- dim.setQueryOrder(queryOrder)
- queryOrder = queryOrder + 1
- selectedDims += dim
- } else {
- val carbonMeasure =
- carbonTable.getMeasureByName(carbonTable.getFactTableName, attr.name)
- if (carbonMeasure != null) {
+ val carbonColumn = carbonTable.getColumnByName(carbonTable.getFactTableName, attr.name)
+ if (carbonColumn != null) {
+ if (carbonColumn.isDimesion()) {
+ val dim = new QueryDimension(attr.name)
+ dim.setQueryOrder(queryOrder)
+ queryOrder = queryOrder + 1
+ selectedDims += dim
+ } else {
val m1 = new QueryMeasure(attr.name)
m1.setQueryOrder(queryOrder)
queryOrder = queryOrder + 1
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 47fa82e..aa9f95a 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -505,6 +505,9 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
tableProperties: Map[String, String]): TableModel
= {
+ fields.zipWithIndex.foreach { x =>
+ x._1.schemaOrdinal = x._2
+ }
val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
fields, tableProperties)
if (dims.isEmpty) {
@@ -1154,36 +1157,36 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
val dataType = field.dataType.getOrElse("NIL")
dataType match {
case "string" => Field(field.column, Some("String"), field.name, Some(null), field.parent,
- field.storeType
+ field.storeType, field.schemaOrdinal
)
case "integer" | "int" => Field(field.column, Some("Integer"), field.name, Some(null),
- field.parent, field.storeType
+ field.parent, field.storeType, field.schemaOrdinal
)
case "long" => Field(field.column, Some("Long"), field.name, Some(null), field.parent,
- field.storeType
+ field.storeType, field.schemaOrdinal
)
case "double" => Field(field.column, Some("Double"), field.name, Some(null), field.parent,
- field.storeType
+ field.storeType, field.schemaOrdinal
)
case "timestamp" => Field(field.column, Some("Timestamp"), field.name, Some(null),
- field.parent, field.storeType
+ field.parent, field.storeType, field.schemaOrdinal
)
case "numeric" => Field(field.column, Some("Numeric"), field.name, Some(null), field.parent,
- field.storeType
+ field.storeType, field.schemaOrdinal
)
case "array" => Field(field.column, Some("Array"), field.name,
field.children.map(f => f.map(normalizeType(_))),
- field.parent, field.storeType
+ field.parent, field.storeType, field.schemaOrdinal
)
case "struct" => Field(field.column, Some("Struct"), field.name,
field.children.map(f => f.map(normalizeType(_))),
- field.parent, field.storeType
+ field.parent, field.storeType, field.schemaOrdinal
)
case "bigint" => Field(field.column, Some("BigInt"), field.name, Some(null), field.parent,
- field.storeType
+ field.storeType, field.schemaOrdinal
)
case "decimal" => Field(field.column, Some("Decimal"), field.name, Some(null), field.parent,
- field.storeType, field.precision, field.scale
+ field.storeType, field.schemaOrdinal, field.precision, field.scale
)
// checking if the nested data type contains the child type as decimal(10,0),
// if it is present then extracting the precision and scale. resetting the data type
@@ -1195,7 +1198,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
field.name,
Some(null),
field.parent,
- field.storeType, precision,
+ field.storeType, field.schemaOrdinal, precision,
scale
)
case _ =>
@@ -1207,10 +1210,10 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
field.dataType.getOrElse("NIL") match {
case "Array" => Field(field.column, Some("Array"), field.name,
field.children.map(f => f.map(appendParentForEachChild(_, field.column))), field.parent,
- field.storeType)
+ field.storeType, field.schemaOrdinal)
case "Struct" => Field(field.column, Some("Struct"), field.name,
field.children.map(f => f.map(appendParentForEachChild(_, field.column))), field.parent,
- field.storeType)
+ field.storeType, field.schemaOrdinal)
case _ => field
}
}
@@ -1243,7 +1246,7 @@ class CarbonSqlParser() extends AbstractSparkSQLParser {
Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName)
case "Decimal" => Field(parentName + "." + field.column, Some("Decimal"),
Some(parentName + "." + field.name.getOrElse(None)), Some(null), parentName,
- field.storeType, field.precision, field.scale)
+ field.storeType, field.schemaOrdinal, field.precision, field.scale)
case _ => field
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 9353a92..1b939f7 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -26,6 +26,7 @@ import scala.language.implicitConversions
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan}
import org.apache.spark.sql.hive.CarbonMetastore
import org.apache.spark.sql.types.TimestampType
@@ -331,6 +332,27 @@ object LoadTable {
}
+private[sql] case class LoadTableByInsert(relation: CarbonDatasourceRelation,
+ child: LogicalPlan) extends RunnableCommand {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ def run(sqlContext: SQLContext): Seq[Row] = {
+ val df = new DataFrame(sqlContext, child)
+ val header = relation.carbonRelation.output.map(_.name).mkString(",")
+ val load = LoadTable(
+ Some(relation.carbonRelation.databaseName),
+ relation.carbonRelation.tableName,
+ null,
+ Seq(),
+ scala.collection.immutable.Map(("fileheader" -> header)),
+ false,
+ null,
+ Some(df)).run(sqlContext)
+ // updating relation metadata. This is in case of auto detect high cardinality
+ relation.carbonRelation.metaData =
+ CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
+ load
+ }
+}
case class LoadTable(
databaseNameOp: Option[String],
tableName: String,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala
new file mode 100644
index 0000000..381697b
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonPreInsertionCasts.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.expressions.Alias
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.expressions.Cast
+import org.apache.spark.sql.catalyst.plans.logical
+import org.apache.spark.sql.catalyst.plans.logical.InsertIntoTable
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules._
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+
+/**
+ * Insert into carbon table from other source
+ */
+object CarbonPreInsertionCasts extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transform {
+ // Wait until children are resolved.
+ case p: LogicalPlan if !p.childrenResolved => p
+
+ case p @ InsertIntoTable(relation: LogicalRelation, _, child, _, _)
+ if relation.relation.isInstanceOf[CarbonDatasourceRelation] =>
+ castChildOutput(p, relation.relation.asInstanceOf[CarbonDatasourceRelation], child)
+ }
+
+ def castChildOutput(p: InsertIntoTable, relation: CarbonDatasourceRelation, child: LogicalPlan)
+ : LogicalPlan = {
+ if (relation.carbonRelation.output.size > CarbonCommonConstants
+ .DEFAULT_MAX_NUMBER_OF_COLUMNS) {
+ sys
+ .error("Maximum supported column by carbon is:" + CarbonCommonConstants
+ .DEFAULT_MAX_NUMBER_OF_COLUMNS
+ )
+ }
+ if (child.output.size >= relation.carbonRelation.output.size ) {
+ InsertIntoCarbonTable(relation, p.partition, p.child, p.overwrite, p.ifNotExists)
+ } else {
+ sys.error("Cannot insert into target table because column number are different")
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
index 44dbd6b..91c96af 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
@@ -67,6 +67,10 @@ class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {
} else {
carbonRawScan(projectList, predicates, l)(sqlContext) :: Nil
}
+ case InsertIntoCarbonTable(relation: CarbonDatasourceRelation,
+ _, child: LogicalPlan, _, _) =>
+ ExecutedCommand(LoadTableByInsert(relation,
+ child)) :: Nil
case CarbonDictionaryCatalystDecoder(relations, profile, aliasMap, _, child) =>
CarbonDictionaryDecoder(relations,
profile,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
index 499be5c..1d52762 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
@@ -92,6 +92,7 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
def isOptimized(plan: LogicalPlan): Boolean = {
plan find {
case cd: CarbonDictionaryCatalystDecoder => true
+ case ic: InsertIntoCarbonTable => true
case other => false
} isDefined
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/resources/shortolap.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/shortolap.csv b/integration/spark/src/test/resources/shortolap.csv
new file mode 100644
index 0000000..1c237b8
--- /dev/null
+++ b/integration/spark/src/test/resources/shortolap.csv
@@ -0,0 +1,5 @@
+imei0,2147,9279,100.05,100.055,2016-05-01 12:25:36,aa,11
+imei1,-2148,-9807,10.05,100.05,2016-05-02 19:25:15,bb,22
+imei2,2147,9279,100.05,100.055,2016-05-01 12:25:36,cc,33
+imei3,-217,-9206,100.005,100.05,2016-05-02 19:25:15,dd,44
+imei4,10,0,15.5,45,2016-05-02 19:25:15,ee,55
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
index bc39df3..1d8eea5 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
@@ -42,10 +42,10 @@ class TestLoadDataWithSingleQuotechar extends QueryTest with BeforeAndAfterAll {
"carbontable OPTIONS('DELIMITER'= ',')")
checkAnswer(
sql("SELECT * from carbontable"),
- Seq(Row("Tom",1),
- Row("Tony\n3,Lily",2),
- Row("Games\"",4),
- Row("prival\"\n6,\"hello\"",5)
+ Seq(Row(1,"Tom"),
+ Row(2,"Tony\n3,Lily"),
+ Row(4,"Games\""),
+ Row(5,"prival\"\n6,\"hello\"")
)
)
} catch {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
new file mode 100644
index 0000000..4199062
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.spark.testsuite.allqueries
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll {
+ sql("drop table if exists THive")
+ sql("create table THive (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions st
ring, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
+ sql("LOAD DATA local INPATH './src/test/resources/100_olap.csv' INTO TABLE THive")
+ }
+ test("insert from hive") {
+ val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions
string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into TCarbon select * from THive")
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from THive order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Late
st_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"),
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La
test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription")
+ )
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
+ }
+ test("insert from hive-sum expression") {
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (MAC string,deviceInformationIdSum int) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into TCarbon select MAC,sum(deviceInformationId+ 10) as a from THive group by MAC")
+ checkAnswer(
+ sql("select MAC,deviceInformationIdSum from TCarbon order by MAC"),
+ sql("select MAC,sum(deviceInformationId+ 10) as a from THive group by MAC order by MAC")
+ )
+ }
+ test("insert from carbon-select columns") {
+ val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ sql("drop table if exists TCarbonSource")
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbonSource (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVe
rsions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonSource options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVer
Number,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions
string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into TCarbon select * from TCarbonSource")
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbonSource order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNum
ber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"),
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La
test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription")
+ )
+ val result = sql("show segments for table TCarbon").collect()(0).get(1).toString()
+ if(!"Success".equalsIgnoreCase(result)) {
+ assert(false)
+ }
+
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
+ }
+ test("insert from carbon-select columns-source table has more column then target column") {
+ val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ sql("drop table if exists load")
+ sql("drop table if exists inser")
+ sql("CREATE TABLE load(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp,name string,point int)STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/shortolap.csv' INTO TABLE load options ('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER' = 'imei,age,task,num,level,productdate,name,point')")
+ sql("CREATE TABLE inser(imei string,age int,task bigint,num double,level decimal(10,3),productdate timestamp)STORED BY 'org.apache.carbondata.format'")
+ sql("insert into inser select * from load")
+ checkAnswer(
+ sql("select * from inser"),
+ sql("select imei,age,task,num,level,productdate from load")
+ )
+ sql("drop table if exists load")
+ sql("drop table if exists inser")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
+ }
+ test("insert from carbon-select * columns") {
+ val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ sql("drop table if exists TCarbonSource")
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbonSource (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVe
rsions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonSource options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVer
Number,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions
string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into TCarbon select * from TCarbonSource")
+ checkAnswer(
+ sql("select * from TCarbonSource"),
+ sql("select * from TCarbon")
+ )
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
+ }
+ test("insert->hive column more than carbon column->success") {
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+
+ sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber,device_backColor,modelId,CUPAudit,CPIClocked from THive")
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from THive"),
+ sql("select imei,deviceInformationId,MAC,deviceColor,gamePointId,contractNumber from TCarbon")
+ )
+ }
+ test("insert->carbon column is more then hive-fails") {
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ try {
+ sql("insert into TCarbon select imei,deviceInformationId,MAC,deviceColor,gamePointId from THive")
+ assert(false)
+ } catch {
+ case ex: Exception => assert(true)
+ }
+ }
+ test("insert->insert wrong data types-pass") {
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
+ sql("drop table if exists TCarbonLocal")
+ sql("create table TCarbonLocal (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbonLocal options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,MAC,deviceInformationId')")
+ sql("insert into TCarbon select imei,MAC,deviceInformationId from THive")
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC from TCarbon"),
+ sql("select imei,deviceInformationId,MAC from TCarbonLocal")
+ )
+ }
+ test("insert->insert empty data -pass") {
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into TCarbon select imei,deviceInformationId,MAC from THive where MAC='wrongdata'")
+ val result = sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'").collect()
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC from THive where MAC='wrongdata'"),
+ sql("select imei,deviceInformationId,MAC from TCarbon where MAC='wrongdata'")
+ )
+ }
+ test("insert into existing load-pass") {
+ val timeStampPropOrig = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+ sql("drop table if exists TCarbon")
+ sql("create table TCarbon (imei string,deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked string,series string,productionDate timestamp,bomCode string,internalModels string, deliveryTime string, channelsId string, channelsName string , deliveryAreaId string, deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string, ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, Active_phonePADPartitionedVersions
string, Latest_YEAR int, Latest_MONTH int, Latest_DAY Decimal(30,10), Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province string, Latest_city string, Latest_district string, Latest_street string, Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string, Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string, Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, Latest_operatorId string, gamePointDescription string,gamePointId double,contractNumber BigInt) STORED BY 'org.apache.carbondata.format'")
+ sql("LOAD DATA INPATH './src/test/resources/100_olap.csv' INTO table TCarbon options ('DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber
,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointDescription,gamePointId,contractNumber')")
+ sql("insert into TCarbon select * from THive")
+ sql("LOAD DATA local INPATH './src/test/resources/100_olap.csv' INTO TABLE THive")
+ checkAnswer(
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from THive order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Late
st_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"),
+ sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La
test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription")
+ )
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
+ }
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS THive")
+ sql("DROP TABLE IF EXISTS TCarbon")
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5f1abef7/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
index 8bcbbbc..17fe053 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestNoInvertedIndexLoadAndQuery.scala
@@ -63,7 +63,7 @@ class TestNoInvertedIndexLoadAndQuery extends QueryTest with BeforeAndAfterAll{
sql("""
SELECT * FROM index1 WHERE city = "Bangalore"
"""),
- Seq(Row("Emily", "Bangalore", 19.0)))
+ Seq(Row(19.0,"Emily", "Bangalore" )))
}
[2/2] incubator-carbondata git commit: [CARBONDATA-368]Insert into
carbon table feature This closes #366
Posted by ja...@apache.org.
[CARBONDATA-368]Insert into carbon table feature This closes #366
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/90bc3669
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/90bc3669
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/90bc3669
Branch: refs/heads/master
Commit: 90bc3669974f136c117b9c3424dc2ebc8bd91738
Parents: 997af85 5f1abef
Author: jackylk <ja...@huawei.com>
Authored: Thu Dec 1 23:29:28 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 1 23:29:28 2016 +0800
----------------------------------------------------------------------
.../ThriftWrapperSchemaConverterImpl.java | 2 +
.../metadata/schema/table/CarbonTable.java | 80 ++++++++-
.../schema/table/column/CarbonColumn.java | 14 +-
.../schema/table/column/CarbonDimension.java | 7 +-
.../schema/table/column/CarbonMeasure.java | 6 +-
.../schema/table/column/ColumnSchema.java | 8 +
.../core/constants/CarbonCommonConstants.java | 9 +
.../carbondata/scan/filter/FilterUtilTest.java | 8 +-
format/src/main/thrift/schema.thrift | 7 +-
.../execution/command/carbonTableSchema.scala | 15 +-
.../spark/sql/CarbonCatalystOperators.scala | 21 +++
.../org/apache/spark/sql/CarbonContext.scala | 2 +-
.../spark/sql/CarbonDatasourceRelation.scala | 43 ++++-
.../scala/org/apache/spark/sql/CarbonScan.scala | 43 ++---
.../org/apache/spark/sql/CarbonSqlParser.scala | 31 ++--
.../execution/command/carbonTableSchema.scala | 22 +++
.../sql/hive/CarbonPreInsertionCasts.scala | 59 ++++++
.../spark/sql/hive/CarbonStrategies.scala | 4 +
.../spark/sql/optimizer/CarbonOptimizer.scala | 1 +
.../spark/src/test/resources/shortolap.csv | 5 +
.../TestLoadDataWithSingleQuotechar.scala | 8 +-
.../InsertIntoCarbonTableTestCase.scala | 178 +++++++++++++++++++
.../TestNoInvertedIndexLoadAndQuery.scala | 2 +-
23 files changed, 500 insertions(+), 75 deletions(-)
----------------------------------------------------------------------