You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/04 14:04:58 UTC
[1/8] carbondata git commit: Rectify Vector Buffer Calculation
Repository: carbondata
Updated Branches:
refs/heads/encoding_override 924f0b7a7 -> b699ee6f7
Rectify Vector Buffer Calculation
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/03d484ab
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/03d484ab
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/03d484ab
Branch: refs/heads/encoding_override
Commit: 03d484abf2cf5c57aeda11ca9355cfbfbdf137c2
Parents: 924f0b7
Author: sounakr <so...@gmail.com>
Authored: Thu Jun 29 01:15:21 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Jul 3 15:06:43 2017 +0530
----------------------------------------------------------------------
.../collector/impl/DictionaryBasedVectorResultCollector.java | 5 +++--
.../collector/impl/RestructureBasedVectorResultCollector.java | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03d484ab/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index 73ccb5d..c857a47 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -139,8 +139,9 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
}
int rowCounter = scannedResult.getRowCounter();
int availableRows = currentPageRowCount - rowCounter;
- int requiredRows =
- columnarBatch.getBatchSize() - (columnarBatch.getActualSize() + filteredRows);
+ // getRowCounter holds total number or rows being placed in Vector. Calculate the
+ // Left over space through getRowCounter only.
+ int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getRowCounter();
requiredRows = Math.min(requiredRows, availableRows);
if (requiredRows < 1) {
return;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03d484ab/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index 6f45c47..8ae0d96 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -103,7 +103,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
}
int rowCounter = scannedResult.getRowCounter();
int availableRows = currentPageRowCount - rowCounter;
- int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getActualSize();
+ int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getRowCounter();
requiredRows = Math.min(requiredRows, availableRows);
if (requiredRows < 1) {
return;
[4/8] carbondata git commit: [CARBONDATA-1253] Sort_columns should
not support float, double, decimal
Posted by ja...@apache.org.
[CARBONDATA-1253] Sort_columns should not support float,double,decimal
This closes #1122
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f9741eb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f9741eb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f9741eb
Branch: refs/heads/encoding_override
Commit: 5f9741ebcb5b2b606f4d710785ce97c4d6b49229
Parents: 0d46976
Author: QiangCai <qi...@qq.com>
Authored: Fri Jun 30 19:51:19 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Jul 3 20:53:20 2017 +0800
----------------------------------------------------------------------
...feVariableLengthDimensionDataChunkStore.java | 12 ---
...afeVariableLengthDimesionDataChunkStore.java | 12 ---
.../apache/carbondata/core/util/ByteUtil.java | 81 --------------------
.../carbondata/core/util/DataTypeUtil.java | 12 ---
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 12 ++-
5 files changed, 10 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index a65d745..2079811 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -25,10 +25,6 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.spark.sql.types.BooleanType;
import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
import org.apache.spark.sql.types.IntegerType;
import org.apache.spark.sql.types.LongType;
import org.apache.spark.sql.types.ShortType;
@@ -157,16 +153,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length));
} else if (dt instanceof IntegerType) {
vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length));
- } else if (dt instanceof FloatType) {
- vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset));
- } else if (dt instanceof DoubleType) {
- vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset));
} else if (dt instanceof LongType) {
vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length));
- } else if (dt instanceof DecimalType) {
- vector.putDecimal(vectorRow,
- Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)),
- DecimalType.MAX_PRECISION());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 03ba34e..c6c98f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -26,10 +26,6 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.spark.sql.types.BooleanType;
import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
import org.apache.spark.sql.types.IntegerType;
import org.apache.spark.sql.types.LongType;
import org.apache.spark.sql.types.ShortType;
@@ -184,16 +180,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore
vector.putShort(vectorRow, ByteUtil.toShort(value, 0, value.length));
} else if (dt instanceof IntegerType) {
vector.putInt(vectorRow, ByteUtil.toInt(value, 0, value.length));
- } else if (dt instanceof FloatType) {
- vector.putFloat(vectorRow, ByteUtil.toFloat(value, 0));
- } else if (dt instanceof DoubleType) {
- vector.putDouble(vectorRow, ByteUtil.toDouble(value, 0));
} else if (dt instanceof LongType) {
vector.putLong(vectorRow, ByteUtil.toLong(value, 0, value.length));
- } else if (dt instanceof DecimalType) {
- vector.putDecimal(vectorRow,
- Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)),
- DecimalType.MAX_PRECISION());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 2761d71..b74f206 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -18,8 +18,6 @@
package org.apache.carbondata.core.util;
import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -39,8 +37,6 @@ public final class ByteUtil {
public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
- public static final byte[] ZERO_IN_BYTES = toBytes(0);
-
private ByteUtil() {
}
@@ -531,28 +527,6 @@ public final class ByteUtil {
}
/**
- * float => byte[]
- *
- * @param f
- * @return
- */
- public static byte[] toBytes(final float f) {
- // Encode it as int
- return toBytes(Float.floatToRawIntBits(f));
- }
-
- /**
- * byte[] => float
- *
- * @param bytes
- * @param offset
- * @return
- */
- public static float toFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
- }
-
- /**
* long => byte[]
*
* @param val
@@ -593,61 +567,6 @@ public final class ByteUtil {
return l ^ Long.MIN_VALUE;
}
- /**
- * doube => byte[]
- *
- * @param d
- * @return
- */
- public static byte[] toBytes(final double d) {
- // Encode it as a long
- return toBytes(Double.doubleToRawLongBits(d));
- }
-
- /**
- * byte[] => double
- *
- * @param bytes
- * @param offset
- * @return
- */
- public static double toDouble(final byte[] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
- }
-
- /**
- * BigDecimal => byte[]
- *
- * @param val
- * @return
- */
- public static byte[] toBytes(BigDecimal val) {
- byte[] valueBytes = val.unscaledValue().toByteArray();
- byte[] result = new byte[valueBytes.length + SIZEOF_INT];
- int offset = putInt(result, 0, val.scale());
- putBytes(result, offset, valueBytes, 0, valueBytes.length);
- return result;
- }
-
- /**
- * byte[] => BigDecimal
- *
- * @param bytes
- * @param offset
- * @param length
- * @return
- */
- public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
- if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
- return null;
- }
-
- int scale = toInt(bytes, offset, bytes.length);
- byte[] tcBytes = new byte[length - SIZEOF_INT];
- System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
- return new BigDecimal(new BigInteger(tcBytes), scale);
- }
-
private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
final int offset, final int length, final int expectedLength) {
String reason;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 1b1884e..37ae5bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -334,14 +334,8 @@ public final class DataTypeUtil {
return ByteUtil.toBytes(Short.parseShort(dimensionValue));
case INT:
return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
- case FLOAT:
- return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
case LONG:
return ByteUtil.toBytes(Long.parseLong(dimensionValue));
- case DOUBLE:
- return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
- case DECIMAL:
- return ByteUtil.toBytes(new BigDecimal(dimensionValue));
default:
return ByteUtil.toBytes(dimensionValue);
}
@@ -372,14 +366,8 @@ public final class DataTypeUtil {
return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
case INT:
return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
- case FLOAT:
- return ByteUtil.toFloat(dataInBytes, 0);
case LONG:
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
- case DOUBLE:
- return ByteUtil.toDouble(dataInBytes, 0);
- case DECIMAL:
- return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
default:
return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index c565c31..8207a9d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -551,8 +551,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
} else {
val dataType = fields.find(x =>
x.column.equalsIgnoreCase(column)).get.dataType.get
- if (isComplexDimDictionaryExclude(dataType)) {
- val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ if (isDataTypeSupportedForSortColumn(dataType)) {
+ val errormsg = s"sort_columns is unsupported for ${dataType} datatype column: " + column
throw new MalformedCarbonCommandException(errormsg)
}
}
@@ -691,6 +691,14 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
+ * detects whether datatype is part of sort_column
+ */
+ private def isDataTypeSupportedForSortColumn(columnDataType: String): Boolean = {
+ val dataTypes = Array("array", "struct", "double", "float", "decimal")
+ dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
+ }
+
+ /**
* detects whether datatype is part of dictionary_exclude
*/
def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {
[5/8] carbondata git commit: Fixed described formatted for
sort_columns after alter
Posted by ja...@apache.org.
Fixed described formatted for sort_columns after alter
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9329ee7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9329ee7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9329ee7
Branch: refs/heads/encoding_override
Commit: e9329ee7c1adc913d6e65c970e6312a5b18c6ec2
Parents: 5f9741e
Author: Ayush Mantri <aa...@gmail.com>
Authored: Thu Jun 29 15:18:20 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Mon Jul 3 20:30:18 2017 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 19 +++++++++++++++++++
.../execution/command/carbonTableSchema.scala | 6 +++---
.../execution/command/carbonTableSchema.scala | 6 +++---
.../AlterTableValidationTestCase.scala | 19 ++++++++++++++++++-
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9329ee7/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index ae97262..16ded57 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -691,6 +691,25 @@ public class CarbonTable implements Serializable {
tableMeasuresMap.put(tableName, visibleMeasures);
}
+ /**
+ * Method to get the list of sort columns
+ *
+ * @param tableName
+ * @return List of Sort column
+ */
+ public List<String> getSortColumns(String tableName) {
+ List<String> sort_columsList = new ArrayList<String>(allDimensions.size());
+ List<CarbonDimension> carbonDimensions = tableDimensionsMap.get(tableName);
+ for (CarbonDimension dim : carbonDimensions) {
+ if (dim.isSortColumn()) {
+ sort_columsList.add(dim.getColName());
+ }
+ }
+ return sort_columsList;
+ }
+
+
+
public int getNumberOfSortColumns() {
return numberOfSortColumns;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9329ee7/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 3477abb..70c8407 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -851,9 +851,9 @@ private[sql] case class DescribeCommandFormatted(
} else {
results ++= Seq(("ADAPTIVE", "", ""))
}
- results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getAllDimensions
- .subList(0, relation.metaData.carbonTable.getNumberOfSortColumns).asScala
- .map(column => column.getColName).mkString(","), ""))
+ results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
+ relation.tableMeta.carbonTableIdentifier.getTableName).asScala
+ .map(column => column).mkString(","), ""))
val dimension = carbonTable
.getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
results ++= getColumnGroups(dimension.asScala.toList)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9329ee7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 8fe4bd7..bb5bdd1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -902,9 +902,9 @@ private[sql] case class DescribeCommandFormatted(
} else {
results ++= Seq(("ADAPTIVE", "", ""))
}
- results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getAllDimensions
- .subList(0, relation.metaData.carbonTable.getNumberOfSortColumns).asScala
- .map(column => column.getColName).mkString(","), ""))
+ results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
+ relation.tableMeta.carbonTableIdentifier.getTableName).asScala
+ .map(column => column).mkString(","), ""))
val dimension = carbonTable
.getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
results ++= getColumnGroups(dimension.asScala.toList)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9329ee7/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index 972b0c2..ae64498 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -423,7 +423,22 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql("alter table Default.uniqdata rename to uniqdata1")
checkAnswer(sql("select * from Default.uniqdata1"), Row(1,"hello"))
}
-
+ test("describe formatted for default sort_columns pre and post alter") {
+ sql("CREATE TABLE defaultSortColumnsWithAlter (empno int, empname String, designation String,role String, doj Timestamp) STORED BY 'org.apache.carbondata.format' " +
+ "tblproperties('DICTIONARY_INCLUDE'='empno','DICTIONARY_EXCLUDE'='role')")
+ sql("alter table defaultSortColumnsWithAlter drop columns (designation)")
+ sql("alter table defaultSortColumnsWithAlter add columns (designation12 String)")
+ checkExistence(sql("describe formatted defaultSortColumnsWithAlter"),true,"SORT_COLUMNS")
+ checkExistence(sql("describe formatted defaultSortColumnsWithAlter"),true,"empno,empname,role,doj")
+ }
+ test("describe formatted for specified sort_columns pre and post alter") {
+ sql("CREATE TABLE specifiedSortColumnsWithAlter (empno int, empname String, designation String,role String, doj Timestamp) STORED BY 'org.apache.carbondata.format' " +
+ "tblproperties('sort_columns'='empno,empname,designation,role,doj','DICTIONARY_INCLUDE'='empno','DICTIONARY_EXCLUDE'='role')")
+ sql("alter table specifiedSortColumnsWithAlter drop columns (designation)")
+ sql("alter table specifiedSortColumnsWithAlter add columns (designation12 String)")
+ checkExistence(sql("describe formatted specifiedSortColumnsWithAlter"),true,"SORT_COLUMNS")
+ checkExistence(sql("describe formatted specifiedSortColumnsWithAlter"),true,"empno,empname,role,doj")
+ }
override def afterAll {
sql("DROP TABLE IF EXISTS restructure")
sql("DROP TABLE IF EXISTS restructure_new")
@@ -432,5 +447,7 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS restructure_badnew")
sql("DROP TABLE IF EXISTS lock_rename")
sql("drop table if exists uniqdata")
+ sql("drop table if exists defaultSortColumnsWithAlter")
+ sql("drop table if exists specifiedSortColumnsWithAlter")
}
}
[8/8] carbondata git commit: [CARBONDATA-1244] Polish docs and
comments in presto integration
Posted by ja...@apache.org.
[CARBONDATA-1244] Polish docs and comments in presto integration
This closes #1131
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b699ee6f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b699ee6f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b699ee6f
Branch: refs/heads/encoding_override
Commit: b699ee6f72d280ff0969663f598769e66d8abdb9
Parents: bbb95ce
Author: bianhq <bi...@gmail.com>
Authored: Tue Jul 4 01:36:42 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Tue Jul 4 11:52:19 2017 +0800
----------------------------------------------------------------------
integration/presto/README.md | 51 ++++--
.../presto/CarbondataConnectorFactory.java | 2 +-
.../presto/impl/CarbonLocalInputSplit.java | 13 +-
.../presto/impl/CarbonTableCacheModel.java | 2 +-
.../presto/impl/CarbonTableReader.java | 154 +++++++++++++++++--
5 files changed, 185 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b699ee6f/integration/presto/README.md
----------------------------------------------------------------------
diff --git a/integration/presto/README.md b/integration/presto/README.md
index 9935478..dc14cb0 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -20,14 +20,10 @@
Please follow the below steps to query carbondata in presto
### Config presto server
-* Download presto server 0.166 : https://repo1.maven.org/maven2/com/facebook/presto/presto-server/
-* Finish configuration as per https://prestodb.io/docs/current/installation/deployment.html
- for example:
+* Download presto server (0.166 is suggested and supported) : https://repo1.maven.org/maven2/com/facebook/presto/presto-server/
+* Finish presto configuration following https://prestodb.io/docs/current/installation/deployment.html.
+ A configuration example:
```
- carbondata.properties:
- connector.name=carbondata
- carbondata-store=/Users/apple/DEMO/presto_test/data
-
config.properties:
coordinator=true
node-scheduler.include-coordinator=true
@@ -57,30 +53,51 @@ Please follow the below steps to query carbondata in presto
node.id=ffffffff-ffff-ffff-ffff-ffffffffffff
node.data-dir=/Users/apple/DEMO/presto_test/data
```
-* config carbondata-connector for presto
+* Config carbondata-connector for presto
- First:compile carbondata-presto integration module
+ Firstly: Compile carbondata, including carbondata-presto integration module
```
$ git clone https://github.com/apache/carbondata
- $ cd carbondata/integration/presto
- $ mvn clean package
+ $ cd carbondata
+ $ mvn -DskipTests -P{spark-version} -Dspark.version={spark-version-number} -Dhadoop.version={hadoop-version-number} clean package
+ ```
+ Replace the spark and hadoop version with the version used in your cluster.
+ For example, if you are using Spark 2.1.0 and Hadoop 2.7.2, you would like to compile using:
+ ```
+ mvn -DskipTests -Pspark-2.1 -Dspark.version=2.1.0 -Dhadoop.version=2.7.2 clean package
+ ```
+
+ Secondly: Create a folder named 'carbondata' under $PRESTO_HOME$/plugin and
+ copy all jars from carbondata/integration/presto/target/carbondata-presto-x.x.x-SNAPSHOT
+ to $PRESTO_HOME$/plugin/carbondata
+
+ Thirdly: Create a carbondata.properties file under $PRESTO_HOME$/etc/catalog/ containing the following contents:
```
- Second:create one folder "carbondata" under ./presto-server-0.166/plugin
- Third:copy all jar from ./carbondata/integration/presto/target/carbondata-presto-x.x.x-SNAPSHOT
- to ./presto-server-0.166/plugin/carbondata
+ connector.name=carbondata
+ carbondata-store={schema-store-path}
+ ```
+ Replace the schema-store-path with the absolute path of the parent directory of the schema.
+ For example, if you have a schema named 'default' stored in hdfs://namenode:9000/test/carbondata/,
+ Then set carbondata-store=hdfs://namenode:9000/test/carbondata
+
+ If you updated the jar balls or configuration files, make sure you have dispatched them
+ to all the presto nodes and restarted the presto servers on the nodes. The updates will not take effect before restarting.
### Generate CarbonData file
-Please refer to quick start : https://github.com/apache/carbondata/blob/master/docs/quick-start-guide.md
+Please refer to quick start: https://github.com/apache/carbondata/blob/master/docs/quick-start-guide.md.
+Load data statement in Spark can be used to create carbondata tables. And then you can easily find the created
+carbondata files.
### Query carbondata in CLI of presto
-* Download presto-cli-0.166-executable.jar
+* Download presto cli client following: https://prestodb.io/docs/current/installation/cli.html
* Start CLI:
```
- $ ./presto-cli-0.166-executable.jar --server localhost:8086 --catalog carbondata --schema default
+ $ ./presto --server localhost:8086 --catalog carbondata --schema default
```
+ Replace the hostname, port and schema name with your own.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b699ee6f/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java
index d97f19e..d557920 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java
@@ -71,7 +71,7 @@ public class CarbondataConnectorFactory implements ConnectorFactory {
ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
ConnectorRecordSetProvider connectorRecordSet =
injector.getInstance(ConnectorRecordSetProvider.class);
- ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class);
+ ConnectorPageSourceProvider connectorPageSource = injector.getInstance(ConnectorPageSourceProvider.class);
return new CarbondataConnector(lifeCycleManager, metadata,
new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), connectorRecordSet,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b699ee6f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
index ba8d9b5..f0a8428 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
@@ -17,19 +17,22 @@
package org.apache.carbondata.presto.impl;
-import java.util.List;
-
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+/**
+ * CarbonLocalInputSplit represents a block, it contains a set of blocklet.
+ */
public class CarbonLocalInputSplit {
private static final long serialVersionUID = 3520344046772190207L;
private String segmentId;
private String path;
- private long start;
- private long length;
- private List<String> locations;
+ private long start; // the start offset of the block in a carbondata file.
+ private long length; // the length of the block.
+ private List<String> locations;// locations are the locations for different replicas.
private short version;
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b699ee6f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
index 45755d1..2a4db14 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableCacheModel.java
@@ -23,7 +23,7 @@ import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.path.CarbonTablePath;
/**
- * Caching Carbon meta(e.g. TableIdentifier, TablePath, TableInfo, CarbonTable) in Class CarbonTableReader
+ * Caching metadata of CarbonData(e.g. TableIdentifier, TablePath, TableInfo, CarbonTable) in Class CarbonTableReader
* to speed up query
*/
public class CarbonTableCacheModel {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b699ee6f/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index c328a64..54832f5 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -72,15 +72,31 @@ import static java.util.Objects.requireNonNull;
* 2:FileFactory, (physic table file)
* 3:CarbonCommonFactory, (offer some )
* 4:DictionaryFactory, (parse dictionary util)
+ *
+ * Currently, it is mainly used to parse metadata of tables under
+ * the configured carbondata-store path and filter the relevant
+ * input splits with given query predicates.
*/
public class CarbonTableReader {
private CarbonTableConfig config;
+
+ /**
+ * The names of the tables under the schema (this.carbonFileList).
+ */
private List<SchemaTableName> tableList;
+
+ /**
+ * carbonFileList represents the store path of the schema, which is configured as carbondata-store
+ * in the CarbonData catalog file ($PRESTO_HOME$/etc/catalog/carbondata.properties).
+ */
private CarbonFile carbonFileList;
private FileFactory.FileType fileType;
- // A cache for Carbon reader
+ /**
+ * A cache for Carbon reader, with this cache,
+ * metadata of a table is only read from file system once.
+ */
private ConcurrentHashMap<SchemaTableName, CarbonTableCacheModel> cc;
@Inject public CarbonTableReader(CarbonTableConfig config) {
@@ -88,9 +104,14 @@ public class CarbonTableReader {
this.cc = new ConcurrentHashMap<>();
}
- // for worker node to initialize carbon metastore
+ /**
+ * For presto worker node to initialize the metadata cache of a table.
+ * @param table the name of the table and schema.
+ * @return
+ */
public CarbonTableCacheModel getCarbonCache(SchemaTableName table) {
if (!cc.containsKey(table)) {
+ // if this table is not cached, try to read the metadata of the table and cache it.
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(
FileFactory.class.getClassLoader())) {
if (carbonFileList == null) {
@@ -110,17 +131,26 @@ public class CarbonTableReader {
else return null;
}
+ /**
+ * Return the schema names under a schema store path (this.carbonFileList).
+ * @return
+ */
public List<String> getSchemaNames() {
return updateSchemaList();
}
- // default PathFilter
+ // default PathFilter, accepts files in carbondata format (with .carbondata extension).
private static final PathFilter DefaultFilter = new PathFilter() {
@Override public boolean accept(Path path) {
return CarbonTablePath.isCarbonDataFile(path.getName());
}
};
+ /**
+ * Get the CarbonFile instance which represents the store path in the configuration, and assign it to
+ * this.carbonFileList.
+ * @return
+ */
public boolean updateCarbonFile() {
if (carbonFileList == null) {
fileType = FileFactory.getFileType(config.getStorePath());
@@ -133,6 +163,10 @@ public class CarbonTableReader {
return true;
}
+ /**
+ * Return the schema names under a schema store path (this.carbonFileList).
+ * @return
+ */
public List<String> updateSchemaList() {
updateCarbonFile();
@@ -143,13 +177,23 @@ public class CarbonTableReader {
} else return ImmutableList.of();
}
+ /**
+ * Get the names of the tables in the given schema.
+ * @param schema name of the schema
+ * @return
+ */
public Set<String> getTableNames(String schema) {
requireNonNull(schema, "schema is null");
return updateTableList(schema);
}
- public Set<String> updateTableList(String dbName) {
- List<CarbonFile> schema = Stream.of(carbonFileList.listFiles()).filter(a -> dbName.equals(a.getName()))
+ /**
+ * Get the names of the tables in the given schema.
+ * @param schemaName name of the schema
+ * @return
+ */
+ public Set<String> updateTableList(String schemaName) {
+ List<CarbonFile> schema = Stream.of(carbonFileList.listFiles()).filter(a -> schemaName.equals(a.getName()))
.collect(Collectors.toList());
if (schema.size() > 0) {
return Stream.of((schema.get(0)).listFiles()).map(a -> a.getName())
@@ -157,6 +201,11 @@ public class CarbonTableReader {
} else return ImmutableSet.of();
}
+ /**
+ * Get the CarbonTable instance of the given table.
+ * @param schemaTableName name of the given table.
+ * @return
+ */
public CarbonTable getTable(SchemaTableName schemaTableName) {
try {
updateSchemaTables();
@@ -170,6 +219,11 @@ public class CarbonTableReader {
return table;
}
+ /**
+ * Find all the tables under the schema store path (this.carbonFileList)
+ * and cache all the table names in this.tableList. Notice that whenever this method
+ * is called, it clears this.tableList and populate the list by reading the files.
+ */
public void updateSchemaTables() {
// update logic determine later
if (carbonFileList == null) {
@@ -185,6 +239,12 @@ public class CarbonTableReader {
}
}
+ /**
+ * Find the table with the given name and build a CarbonTable instance for it.
+ * This method should be called after this.updateSchemaTables().
+ * @param schemaTableName name of the given table.
+ * @return
+ */
private CarbonTable loadTableMetadata(SchemaTableName schemaTableName) {
for (SchemaTableName table : tableList) {
if (!table.equals(schemaTableName)) continue;
@@ -195,7 +255,9 @@ public class CarbonTableReader {
}
/**
- * parse carbon metadata into cc(CarbonTableReader cache)
+ * Read the metadata of the given table and cache it in this.cc (CarbonTableReader cache).
+ * @param table name of the given table.
+ * @return the CarbonTable instance which contains all the needed metadata for a table.
*/
public CarbonTable parseCarbonMetadata(SchemaTableName table) {
CarbonTable result = null;
@@ -203,17 +265,25 @@ public class CarbonTableReader {
CarbonTableCacheModel cache = cc.getOrDefault(table, new CarbonTableCacheModel());
if (cache.isValid()) return cache.carbonTable;
- //Step1: get table meta path, load carbon table param
+ // If table is not previously cached, then:
+
+ // Step 1: get store path of the table and cache it.
String storePath = config.getStorePath();
+ // create table identifier. the table id is randomly generated.
cache.carbonTableIdentifier =
new CarbonTableIdentifier(table.getSchemaName(), table.getTableName(),
UUID.randomUUID().toString());
+ // get the store path of the table.
cache.carbonTablePath =
PathFactory.getInstance().getCarbonTablePath(storePath, cache.carbonTableIdentifier);
+ // cache the table
cc.put(table, cache);
- //Step2: check file existed? read schema file
+ //Step 2: read the metadata (tableInfo) of the table.
ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
+ // TBase is used to read and write thrift objects.
+ // TableInfo is a kind of TBase used to read and write table information.
+ // TableInfo is generated by thrift, see schema.thrift under format/src/main/thrift for details.
public TBase create() {
return new org.apache.carbondata.format.TableInfo();
}
@@ -225,14 +295,16 @@ public class CarbonTableReader {
(org.apache.carbondata.format.TableInfo) thriftReader.read();
thriftReader.close();
- // Step3: Transform Format Level TableInfo to Code Level TableInfo
+ // Step 3: convert format level TableInfo to code level TableInfo
SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ // wrapperTableInfo is the code level information of a table in carbondata core, different from the Thrift TableInfo.
TableInfo wrapperTableInfo = schemaConverter
.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(),
storePath);
wrapperTableInfo.setMetaDataFilepath(
CarbonTablePath.getFolderContainingFile(cache.carbonTablePath.getSchemaFilePath()));
- // Step4: Load metadata info into CarbonMetadata
+
+ // Step 4: Load metadata info into CarbonMetadata
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
cache.tableInfo = wrapperTableInfo;
@@ -246,6 +318,13 @@ public class CarbonTableReader {
return result;
}
+ /**
+ * Apply filters to the table and get valid input splits of the table.
+ * @param tableCacheModel the table
+ * @param filters the filters
+ * @return
+ * @throws Exception
+ */
public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel,
Expression filters) throws Exception {
@@ -332,7 +411,16 @@ public class CarbonTableReader {
}
/**
- * get data blocks of given segment
+ * Get all the data blocks of a given segment.
+ * @param filterExpressionProcessor
+ * @param absoluteTableIdentifier
+ * @param tablePath
+ * @param resolver
+ * @param segmentId
+ * @param cacheClient
+ * @param updateStatusManager
+ * @return
+ * @throws IOException
*/
private List<DataRefNode> getDataBlocksOfSegment(
FilterExpressionProcessor filterExpressionProcessor,
@@ -380,6 +468,16 @@ public class CarbonTableReader {
return false;
}
+ /**
+ * Build and load the B-trees of the segment.
+ * @param absoluteTableIdentifier
+ * @param tablePath
+ * @param segmentId
+ * @param cacheClient
+ * @param updateStatusManager
+ * @return
+ * @throws IOException
+ */
private Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> getSegmentAbstractIndexs(/*JobContext job,*/
AbsoluteTableIdentifier absoluteTableIdentifier, CarbonTablePath tablePath, String segmentId,
CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
@@ -491,6 +589,13 @@ public class CarbonTableReader {
return false;
}
+ /**
+ * Get the input splits of a set of carbondata files.
+ * @param fileStatusList the file statuses of the set of carbondata files.
+ * @param targetSystem hdfs FileSystem
+ * @return
+ * @throws IOException
+ */
private List<InputSplit> getSplit(List<FileStatus> fileStatusList, FileSystem targetSystem)
throws IOException {
@@ -501,6 +606,7 @@ public class CarbonTableReader {
while (true) {
while (true) {
while (split.hasNext()) {
+ // file is a carbondata file
FileStatus file = (FileStatus) split.next();
Path path = file.getPath();
long length = file.getLen();
@@ -520,7 +626,7 @@ public class CarbonTableReader {
int blkIndex;
for (
bytesRemaining = length;
- (double) bytesRemaining / (double) splitSize > 1.1D;
+ (double) bytesRemaining / (double) splitSize > 1.1D;// when there are more than one splits left.
bytesRemaining -= splitSize) {
blkIndex = this.getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(this.makeSplit(path, length - bytesRemaining, splitSize,
@@ -552,6 +658,15 @@ public class CarbonTableReader {
return new String[] { "0" };
}
+ /**
+ * Get all file statuses of the carbondata files with a segmentId in segmentsToConsider
+ * under the tablePath, and add them to the result.
+ * @param segmentsToConsider
+ * @param tablePath
+ * @param result
+ * @return the FileSystem instance been used in this function.
+ * @throws IOException
+ */
private FileSystem getFileStatusOfSegments(String[] segmentsToConsider, CarbonTablePath tablePath,
List<FileStatus> result) throws IOException {
String[] partitionsToConsider = getValidPartitions();
@@ -584,6 +699,7 @@ public class CarbonTableReader {
LocatedFileStatus stat = iter.next();
if (DefaultFilter.accept(stat.getPath())) {
if (stat.isDirectory()) {
+ // DefaultFiler accepts carbondata files.
addInputPathRecursively(result, fs, stat.getPath(), DefaultFilter);
} else {
result.add(stat);
@@ -598,6 +714,15 @@ public class CarbonTableReader {
return fs;
}
+ /**
+ * Get the FileStatus of all carbondata files under the path recursively,
+ * and add the file statuses into the result
+ * @param result
+ * @param fs
+ * @param path
+ * @param inputFilter the filter used to determinate whether a path is a carbondata file
+ * @throws IOException
+ */
protected void addInputPathRecursively(List<FileStatus> result, FileSystem fs, Path path,
PathFilter inputFilter) throws IOException {
RemoteIterator iter = fs.listLocatedStatus(path);
@@ -616,7 +741,10 @@ public class CarbonTableReader {
}
/**
- * get data blocks of given btree
+ * Get the data blocks of a b tree. the root node of the b tree is abstractIndex.dataRefNode.
+ * BTreeNode is a sub class of DataRefNode.
+ * @param abstractIndex
+ * @return
*/
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
List<DataRefNode> blocks = new LinkedList<DataRefNode>();
[2/8] carbondata git commit: [CARBONDATA-1256] Rectify Vector Buffer
Overflow Calculation. This closes #1111
Posted by ja...@apache.org.
[CARBONDATA-1256] Rectify Vector Buffer Overflow Calculation. This closes #1111
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27d520cc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27d520cc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27d520cc
Branch: refs/heads/encoding_override
Commit: 27d520cc0be71ab23c705e8c2e2a8fdcdbe589be
Parents: 924f0b7 03d484a
Author: ravipesala <ra...@gmail.com>
Authored: Mon Jul 3 15:07:37 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon Jul 3 15:07:37 2017 +0530
----------------------------------------------------------------------
.../collector/impl/DictionaryBasedVectorResultCollector.java | 5 +++--
.../collector/impl/RestructureBasedVectorResultCollector.java | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[7/8] carbondata git commit: [CARBONDATA-980] Fix for Is Not Null in
presto
Posted by ja...@apache.org.
[CARBONDATA-980] Fix for Is Not Null in presto
This closes #1128
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bbb95ceb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bbb95ceb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bbb95ceb
Branch: refs/heads/encoding_override
Commit: bbb95ceb4041b818982214c395637c10d3904912
Parents: 0140a12
Author: jatin <ja...@knoldus.in>
Authored: Mon Jul 3 18:08:22 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Mon Jul 3 23:35:07 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/presto/CarbondataRecordSetProvider.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb95ceb/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
index 71649f3..4b7864d 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
@@ -144,7 +144,6 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
List<Object> singleValues = new ArrayList<>();
List<Expression> disjuncts = new ArrayList<>();
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
- checkState(!range.isAll()); // Already checked
if (range.isSingleValue()) {
singleValues.add(range.getLow().getValue());
} else {
[6/8] carbondata git commit: [CARBONDATA-1254] Fixed describe
formatted for sort columns after alter #1123
Posted by ja...@apache.org.
[CARBONDATA-1254] Fixed describe formatted for sort columns after alter #1123
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0140a12e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0140a12e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0140a12e
Branch: refs/heads/encoding_override
Commit: 0140a12e6b34bbc36838221d1117553d3644bcea
Parents: 5f9741e e9329ee
Author: kumarvishal <ku...@gmail.com>
Authored: Mon Jul 3 20:31:45 2017 +0530
Committer: kumarvishal <ku...@gmail.com>
Committed: Mon Jul 3 20:31:45 2017 +0530
----------------------------------------------------------------------
.../core/metadata/schema/table/CarbonTable.java | 19 +++++++++++++++++++
.../execution/command/carbonTableSchema.scala | 6 +++---
.../execution/command/carbonTableSchema.scala | 6 +++---
.../AlterTableValidationTestCase.scala | 19 ++++++++++++++++++-
4 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[3/8] carbondata git commit: [CARBONDATA-982] Fixed Bug For NotIn
Clause In Presto
Posted by ja...@apache.org.
[CARBONDATA-982] Fixed Bug For NotIn Clause In Presto
This closes #1062
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0d469761
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0d469761
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0d469761
Branch: refs/heads/encoding_override
Commit: 0d46976105239dd9ab94b83c3f0ca287b34fee8d
Parents: 27d520c
Author: Geetika gupta <ge...@knoldus.in>
Authored: Fri Jun 16 13:07:52 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Mon Jul 3 17:43:56 2017 +0800
----------------------------------------------------------------------
.../carbondata/presto/CarbondataRecordSet.java | 5 +-
.../presto/CarbondataRecordSetProvider.java | 42 ++++-----
.../presto/CarbondataSplitManager.java | 96 ++++++++++++--------
3 files changed, 84 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0d469761/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
index a28342e..d75cbfb 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
@@ -21,6 +21,7 @@ import com.facebook.presto.spi.*;
import com.facebook.presto.spi.predicate.TupleDomain;
import com.facebook.presto.spi.type.Type;
import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -83,9 +84,9 @@ public class CarbondataRecordSet implements RecordSet {
tableBlockInfoList.add(new TableBlockInfo(split.getLocalInputSplit().getPath().toString(),
split.getLocalInputSplit().getStart(), split.getLocalInputSplit().getSegmentId(),
split.getLocalInputSplit().getLocations().toArray(new String[0]),
- split.getLocalInputSplit().getLength(),
+ split.getLocalInputSplit().getLength(),new BlockletInfos(),
//blockletInfos,
- ColumnarFormatVersion.valueOf(split.getLocalInputSplit().getVersion()), null));
+ ColumnarFormatVersion.valueOf(split.getLocalInputSplit().getVersion()),null));
queryModel.setTableBlockInfos(tableBlockInfoList);
queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0d469761/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
index a9652cc..71649f3 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
@@ -142,13 +142,13 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
}
List<Object> singleValues = new ArrayList<>();
- List<Expression> rangeFilter = new ArrayList<>();
+ List<Expression> disjuncts = new ArrayList<>();
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
checkState(!range.isAll()); // Already checked
if (range.isSingleValue()) {
singleValues.add(range.getLow().getValue());
} else {
- List<String> rangeConjuncts = new ArrayList<>();
+ List<Expression> rangeConjuncts = new ArrayList<>();
if (!range.getLow().isLowerUnbounded()) {
Object value = ConvertDataByType(range.getLow().getValue(), type);
switch (range.getLow().getBound()) {
@@ -157,15 +157,15 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
//todo not now
} else {
GreaterThanExpression greater = new GreaterThanExpression(colExpression,
- new LiteralExpression(value, coltype));
- rangeFilter.add(greater);
+ new LiteralExpression(value, coltype));
+ rangeConjuncts.add(greater);
}
break;
case EXACTLY:
GreaterThanEqualToExpression greater =
- new GreaterThanEqualToExpression(colExpression,
- new LiteralExpression(value, coltype));
- rangeFilter.add(greater);
+ new GreaterThanEqualToExpression(colExpression,
+ new LiteralExpression(value, coltype));
+ rangeConjuncts.add(greater);
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
@@ -180,21 +180,21 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
throw new IllegalArgumentException("High marker should never use ABOVE bound");
case EXACTLY:
LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression,
- new LiteralExpression(value, coltype));
- rangeFilter.add(less);
+ new LiteralExpression(value, coltype));
+ rangeConjuncts.add(less);
break;
case BELOW:
LessThanExpression less2 =
- new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
- rangeFilter.add(less2);
+ new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
+ rangeConjuncts.add(less2);
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
}
}
+ disjuncts.addAll(rangeConjuncts);
}
}
-
if (singleValues.size() == 1) {
Expression ex = null;
if (coltype.equals(DataType.STRING)) {
@@ -215,25 +215,25 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
candidates = new ListExpression(exs);
if (candidates != null) filters.add(new InExpression(colExpression, candidates));
- } else if (rangeFilter.size() > 0) {
- if (rangeFilter.size() > 1) {
- Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
- if (rangeFilter.size() > 2) {
- for (int i = 2; i < rangeFilter.size(); i++) {
- filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
+ } else if (disjuncts.size() > 0) {
+ if (disjuncts.size() > 1) {
+ Expression finalFilters = new OrExpression(disjuncts.get(0), disjuncts.get(1));
+ if (disjuncts.size() > 2) {
+ for (int i = 2; i < disjuncts.size(); i++) {
+ filters.add(new AndExpression(finalFilters, disjuncts.get(i)));
}
}
- } else if (rangeFilter.size() == 1) filters.add(rangeFilter.get(0));
+ } else if (disjuncts.size() == 1) filters.add(disjuncts.get(0));
}
}
Expression finalFilters;
List<Expression> tmp = filters.build();
if (tmp.size() > 1) {
- finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
+ finalFilters = new OrExpression(tmp.get(0), tmp.get(1));
if (tmp.size() > 2) {
for (int i = 2; i < tmp.size(); i++) {
- finalFilters = new AndExpression(finalFilters, tmp.get(i));
+ finalFilters = new OrExpression(finalFilters, tmp.get(i));
}
}
} else if (tmp.size() == 1) finalFilters = tmp.get(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0d469761/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
index e39ee58..0ce0600 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
@@ -17,38 +17,59 @@
package org.apache.carbondata.presto;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
+import org.apache.carbondata.core.scan.expression.conditional.InExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+import org.apache.carbondata.core.scan.expression.logical.OrExpression;
import org.apache.carbondata.presto.impl.CarbonLocalInputSplit;
import org.apache.carbondata.presto.impl.CarbonTableCacheModel;
import org.apache.carbondata.presto.impl.CarbonTableReader;
-import com.facebook.presto.spi.*;
+
+import com.facebook.presto.spi.ColumnHandle;
+import com.facebook.presto.spi.ConnectorSession;
+import com.facebook.presto.spi.ConnectorSplit;
+import com.facebook.presto.spi.ConnectorSplitSource;
+import com.facebook.presto.spi.ConnectorTableLayoutHandle;
+import com.facebook.presto.spi.FixedSplitSource;
+import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
import com.facebook.presto.spi.predicate.Domain;
import com.facebook.presto.spi.predicate.Range;
import com.facebook.presto.spi.predicate.TupleDomain;
-import com.facebook.presto.spi.type.*;
+import com.facebook.presto.spi.type.BigintType;
+import com.facebook.presto.spi.type.BooleanType;
+import com.facebook.presto.spi.type.DateType;
+import com.facebook.presto.spi.type.DecimalType;
+import com.facebook.presto.spi.type.DoubleType;
+import com.facebook.presto.spi.type.IntegerType;
+import com.facebook.presto.spi.type.SmallintType;
+import com.facebook.presto.spi.type.TimestampType;
+import com.facebook.presto.spi.type.Type;
+import com.facebook.presto.spi.type.VarcharType;
import com.google.common.collect.ImmutableList;
import io.airlift.slice.Slice;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.scan.expression.ColumnExpression;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.LiteralExpression;
-import org.apache.carbondata.core.scan.expression.conditional.*;
-import org.apache.carbondata.core.scan.expression.logical.AndExpression;
-import org.apache.carbondata.core.scan.expression.logical.OrExpression;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-import static org.apache.carbondata.presto.Types.checkType;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;
+import static org.apache.carbondata.presto.Types.checkType;
/**
* Build Carbontable splits
@@ -152,13 +173,14 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
}
List<Object> singleValues = new ArrayList<>();
- List<Expression> rangeFilter = new ArrayList<>();
+
+ List<Expression> disjuncts = new ArrayList<>();
+
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
- checkState(!range.isAll()); // Already checked
if (range.isSingleValue()) {
singleValues.add(range.getLow().getValue());
} else {
- List<String> rangeConjuncts = new ArrayList<>();
+ List<Expression> rangeConjuncts = new ArrayList<>();
if (!range.getLow().isLowerUnbounded()) {
Object value = ConvertDataByType(range.getLow().getValue(), type);
switch (range.getLow().getBound()) {
@@ -168,14 +190,14 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
} else {
GreaterThanExpression greater = new GreaterThanExpression(colExpression,
new LiteralExpression(value, coltype));
- rangeFilter.add(greater);
+ rangeConjuncts.add(greater);
}
break;
case EXACTLY:
GreaterThanEqualToExpression greater =
new GreaterThanEqualToExpression(colExpression,
new LiteralExpression(value, coltype));
- rangeFilter.add(greater);
+ rangeConjuncts.add(greater);
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
@@ -191,17 +213,18 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
case EXACTLY:
LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression,
new LiteralExpression(value, coltype));
- rangeFilter.add(less);
+ rangeConjuncts.add(less);
break;
case BELOW:
LessThanExpression less2 =
new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
- rangeFilter.add(less2);
+ rangeConjuncts.add(less2);
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
}
}
+ disjuncts.addAll(rangeConjuncts);
}
}
@@ -221,26 +244,26 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
candidates = new ListExpression(exs);
if (candidates != null) filters.add(new InExpression(colExpression, candidates));
- } else if (rangeFilter.size() > 0) {
- if (rangeFilter.size() > 1) {
- Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
- if (rangeFilter.size() > 2) {
- for (int i = 2; i < rangeFilter.size(); i++) {
- filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
+ } else if (disjuncts.size() > 0) {
+ if (disjuncts.size() > 1) {
+ Expression finalFilters = new OrExpression(disjuncts.get(0), disjuncts.get(1));
+ if (disjuncts.size() > 2) {
+ for (int i = 2; i < disjuncts.size(); i++) {
+ filters.add(new AndExpression(finalFilters, disjuncts.get(i)));
}
}
- } else if (rangeFilter.size() == 1)//only have one value
- filters.add(rangeFilter.get(0));
+ } else if (disjuncts.size() == 1)//only have one value
+ filters.add(disjuncts.get(0));
}
}
Expression finalFilters;
List<Expression> tmp = filters.build();
if (tmp.size() > 1) {
- finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
+ finalFilters = new OrExpression(tmp.get(0), tmp.get(1));
if (tmp.size() > 2) {
for (int i = 2; i < tmp.size(); i++) {
- finalFilters = new AndExpression(finalFilters, tmp.get(i));
+ finalFilters = new OrExpression(finalFilters, tmp.get(i));
}
}
} else if (tmp.size() == 1) finalFilters = tmp.get(0);
@@ -252,6 +275,7 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
/**
* Convert presto spi Type into Carbondata Type
+ *
* @param colType
* @return
*/