You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/11 01:43:03 UTC
[01/18] carbondata git commit: single_pass blocked for global_sort
Repository: carbondata
Updated Branches:
refs/heads/encoding_override b699ee6f7 -> 403c3d9b4
single_pass blocked for global_sort
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a4083bf1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a4083bf1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a4083bf1
Branch: refs/heads/encoding_override
Commit: a4083bf1ada27d257ce2f666018ef361c7f60c1d
Parents: b699ee6
Author: rahulforallp <ra...@knoldus.in>
Authored: Fri Jun 30 00:06:56 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 09:00:16 2017 +0530
----------------------------------------------------------------------
.../execution/command/carbonTableSchema.scala | 1 +
.../execution/command/carbonTableSchema.scala | 221 +++++++++++--------
.../store/CarbonFactDataHandlerColumnar.java | 8 +
3 files changed, 137 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a4083bf1/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index fa5a0d6..ee77f35 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -450,6 +450,7 @@ class TableNewProcessor(cm: TableModel) {
// Setting the boolean value of useInvertedIndex in column schema
val noInvertedIndexCols = cm.noInvertedIdxCols.getOrElse(Seq())
+ LOGGER.info("NoINVERTEDINDEX columns are : " + noInvertedIndexCols.mkString(","))
for (column <- allColumns) {
// When the column is measure or the specified no inverted index column in DDL,
// set useInvertedIndex to false, otherwise true.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a4083bf1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index bb5bdd1..8e7db45 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -356,6 +356,84 @@ case class LoadTable(
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ private def getFinalOptions(carbonProperty: CarbonProperties): scala.collection
+ .mutable.Map[String, String] = {
+ var optionsFinal = scala.collection.mutable.Map[String, String]()
+ optionsFinal.put("delimiter", options.getOrElse("delimiter", ","))
+ optionsFinal.put("quotechar", options.getOrElse("quotechar", "\""))
+ optionsFinal.put("fileheader", options.getOrElse("fileheader", ""))
+ optionsFinal.put("escapechar", options.getOrElse("escapechar", "\\"))
+ optionsFinal.put("commentchar", options.getOrElse("commentchar", "#"))
+ optionsFinal.put("columndict", options.getOrElse("columndict", null))
+ optionsFinal
+ .put("serialization_null_format", options.getOrElse("serialization_null_format", "\\N"))
+ optionsFinal.put("bad_records_logger_enable", options.getOrElse("bad_records_logger_enable",
+ carbonProperty
+ .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)))
+ val badRecordActionValue = carbonProperty
+ .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
+ optionsFinal.put("bad_records_action", options.getOrElse("bad_records_action", carbonProperty
+ .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ badRecordActionValue)))
+ optionsFinal
+ .put("is_empty_data_bad_record", options.getOrElse("is_empty_data_bad_record", carbonProperty
+ .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)))
+ optionsFinal.put("all_dictionary_path", options.getOrElse("all_dictionary_path", ""))
+ optionsFinal
+ .put("complex_delimiter_level_1", options.getOrElse("complex_delimiter_level_1", "\\$"))
+ optionsFinal
+ .put("complex_delimiter_level_2", options.getOrElse("complex_delimiter_level_2", "\\:"))
+ optionsFinal.put("dateformat", options.getOrElse("dateformat",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)))
+
+ optionsFinal.put("global_sort_partitions", options.getOrElse("global_sort_partitions",
+ carbonProperty
+ .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS, null)))
+
+ optionsFinal.put("maxcolumns", options.getOrElse("maxcolumns", null))
+ optionsFinal.put("sort_scope", options
+ .getOrElse("sort_scope",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
+ carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
+ CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))))
+
+ optionsFinal.put("batch_sort_size_inmb", options.getOrElse("batch_sort_size_inmb",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ carbonProperty.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))))
+ optionsFinal.put("bad_record_path", options.getOrElse("bad_record_path",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ carbonProperty.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))))
+
+ val useOnePass = options.getOrElse("single_pass",
+ carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim.toLowerCase match {
+ case "true" =>
+ true
+ case "false" =>
+ // when single_pass = false and if either alldictionarypath
+ // or columnDict is configured the do not allow load
+ if (StringUtils.isNotEmpty(optionsFinal.get("all_dictionary_path").get) ||
+ StringUtils.isNotEmpty(optionsFinal.get("columndict").get)) {
+ throw new MalformedCarbonCommandException(
+ "Can not use all_dictionary_path or columndict without single_pass.")
+ } else {
+ false
+ }
+ case illegal =>
+ LOGGER.error(s"Can't use single_pass, because illegal syntax found: [" + illegal + "] " +
+ "Please set it as 'true' or 'false'")
+ false
+ }
+ optionsFinal.put("single_pass", useOnePass.toString)
+ optionsFinal
+ }
+
private def checkDefaultValue(value: String, default: String) = {
if (StringUtils.isEmpty(value)) {
default
@@ -390,6 +468,7 @@ case class LoadTable(
val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
carbonProperty.addProperty("zookeeper.enable.lock", "false")
+ val optionsFinal = getFinalOptions(carbonProperty)
val carbonLock = CarbonLockFactory
.getCarbonLockObj(relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
.getCarbonTableIdentifier,
@@ -426,66 +505,39 @@ case class LoadTable(
val partitionLocation = relation.tableMeta.storePath + "/partition/" +
relation.tableMeta.carbonTableIdentifier.getDatabaseName + "/" +
relation.tableMeta.carbonTableIdentifier.getTableName + "/"
-
-
val columnar = sparkSession.conf.get("carbon.is.columnar.storage", "true").toBoolean
-
- val delimiter = options.getOrElse("delimiter", ",")
- val quoteChar = options.getOrElse("quotechar", "\"")
- val fileHeader = options.getOrElse("fileheader", "")
- val escapeChar = options.getOrElse("escapechar", "\\")
- val commentChar = options.getOrElse("commentchar", "#")
- val columnDict = options.getOrElse("columndict", null)
- val serializationNullFormat = options.getOrElse("serialization_null_format", "\\N")
- val badRecordsLoggerEnable = options.getOrElse("bad_records_logger_enable",
- carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT))
- val badRecordActionValue = carbonProperty
- .getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
- val badRecordsAction = options.getOrElse("bad_records_action", carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- badRecordActionValue))
- val isEmptyDataBadRecord = options.getOrElse("is_empty_data_bad_record", carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT))
- val allDictionaryPath = options.getOrElse("all_dictionary_path", "")
- val complex_delimiter_level_1 = options.getOrElse("complex_delimiter_level_1", "\\$")
- val complex_delimiter_level_2 = options.getOrElse("complex_delimiter_level_2", "\\:")
- val dateFormat = options.getOrElse("dateformat",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT))
+ val sort_scope = optionsFinal.get("sort_scope").get
+ val single_pass = optionsFinal.get("single_pass").get
+ val bad_records_logger_enable = optionsFinal.get("bad_records_logger_enable").get
+ val bad_records_action = optionsFinal.get("bad_records_action").get
+ val bad_record_path = optionsFinal.get("bad_record_path").get
+ val global_sort_partitions = optionsFinal.get("global_sort_partitions").get
+ val dateFormat = optionsFinal.get("dateformat").get
+ val delimeter = optionsFinal.get("delimiter").get
+ val complex_delimeter_level1 = optionsFinal.get("complex_delimiter_level_1").get
+ val complex_delimeter_level2 = optionsFinal.get("complex_delimiter_level_2").get
+ val all_dictionary_path = optionsFinal.get("all_dictionary_path").get
+ val column_dict = optionsFinal.get("columndict").get
+ if (sort_scope.equals("GLOBAL_SORT") &&
+ single_pass.equals("TRUE")) {
+ sys.error("Global_Sort can't be used with single_pass flow")
+ }
ValidateUtil.validateDateFormat(dateFormat, table, tableName)
- val maxColumns = options.getOrElse("maxcolumns", null)
- val sortScope = options
- .getOrElse("sort_scope",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
- carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
- CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)))
- ValidateUtil.validateSortScope(table, sortScope)
- val batchSortSizeInMB = options.getOrElse("batch_sort_size_inmb",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperty.getProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT)))
- val bad_record_path = options.getOrElse("bad_record_path",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperty.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL)))
- if (badRecordsLoggerEnable.toBoolean ||
- LoggerAction.REDIRECT.name().equalsIgnoreCase(badRecordsAction)) {
+ ValidateUtil.validateSortScope(table, sort_scope)
+
+
+ if (bad_records_logger_enable.toBoolean ||
+ LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
sys.error("Invalid bad records location.")
}
}
carbonLoadModel.setBadRecordsLocation(bad_record_path)
- val globalSortPartitions = options.getOrElse("global_sort_partitions",
- carbonProperty
- .getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS, null))
- ValidateUtil.validateGlobalSortPartitions(globalSortPartitions)
- carbonLoadModel.setEscapeChar(checkDefaultValue(escapeChar, "\\"))
- carbonLoadModel.setQuoteChar(checkDefaultValue(quoteChar, "\""))
- carbonLoadModel.setCommentChar(checkDefaultValue(commentChar, "#"))
+
+ ValidateUtil.validateGlobalSortPartitions(global_sort_partitions)
+ carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal.get("escapechar").get, "\\"))
+ carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal.get("quotechar").get, "\""))
+ carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal.get("commentchar").get, "#"))
carbonLoadModel.setDateFormat(dateFormat)
carbonLoadModel.setDefaultTimestampFormat(carbonProperty.getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
@@ -495,65 +547,48 @@ case class LoadTable(
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
carbonLoadModel
.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," + serializationNullFormat)
+ TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," +
+ optionsFinal.get("serialization_null_format").get)
carbonLoadModel
.setBadRecordsLoggerEnable(
- TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + badRecordsLoggerEnable)
+ TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + bad_records_logger_enable)
carbonLoadModel
.setBadRecordsAction(
- TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + badRecordsAction)
+ TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action)
carbonLoadModel
.setIsEmptyDataBadRecord(
- DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," + isEmptyDataBadRecord)
- carbonLoadModel.setSortScope(sortScope)
- carbonLoadModel.setBatchSortSizeInMb(batchSortSizeInMB)
- carbonLoadModel.setGlobalSortPartitions(globalSortPartitions)
- val useOnePass = options.getOrElse("single_pass",
- carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim.toLowerCase match {
- case "true" =>
- true
- case "false" =>
- // when single_pass = false and if either alldictionarypath
- // or columnDict is configured the do not allow load
- if (StringUtils.isNotEmpty(allDictionaryPath) || StringUtils.isNotEmpty(columnDict)) {
- throw new MalformedCarbonCommandException(
- "Can not use all_dictionary_path or columndict without single_pass.")
- } else {
- false
- }
- case illegal =>
- LOGGER.error(s"Can't use single_pass, because illegal syntax found: [" + illegal + "] " +
- "Please set it as 'true' or 'false'")
- false
- }
- carbonLoadModel.setUseOnePass(useOnePass)
- if (delimiter.equalsIgnoreCase(complex_delimiter_level_1) ||
- complex_delimiter_level_1.equalsIgnoreCase(complex_delimiter_level_2) ||
- delimiter.equalsIgnoreCase(complex_delimiter_level_2)) {
+ DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
+ optionsFinal.get("is_empty_data_bad_record").get)
+ carbonLoadModel.setSortScope(sort_scope)
+ carbonLoadModel.setBatchSortSizeInMb(optionsFinal.get("batch_sort_size_inmb").get)
+ carbonLoadModel.setGlobalSortPartitions(global_sort_partitions)
+ carbonLoadModel.setUseOnePass(single_pass.toBoolean)
+ if (delimeter.equalsIgnoreCase(complex_delimeter_level1) ||
+ complex_delimeter_level1.equalsIgnoreCase(complex_delimeter_level2) ||
+ delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
sys.error(s"Field Delimiter & Complex types delimiter are same")
}
else {
carbonLoadModel.setComplexDelimiterLevel1(
- CarbonUtil.delimiterConverter(complex_delimiter_level_1))
+ CarbonUtil.delimiterConverter(complex_delimeter_level1))
carbonLoadModel.setComplexDelimiterLevel2(
- CarbonUtil.delimiterConverter(complex_delimiter_level_2))
+ CarbonUtil.delimiterConverter(complex_delimeter_level2))
}
// set local dictionary path, and dictionary file extension
- carbonLoadModel.setAllDictPath(allDictionaryPath)
+ carbonLoadModel.setAllDictPath(all_dictionary_path)
val partitionStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
try {
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
carbonLoadModel.setFactFilePath(factPath)
- carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimiter))
- carbonLoadModel.setCsvHeader(fileHeader)
- carbonLoadModel.setColDictFilePath(columnDict)
+ carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter))
+ carbonLoadModel.setCsvHeader(optionsFinal.get("fileheader").get)
+ carbonLoadModel.setColDictFilePath(column_dict)
carbonLoadModel.setDirectLoad(true)
carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
val validatedMaxColumns = CommonUtil.validateMaxColumns(carbonLoadModel.getCsvHeaderColumns,
- maxColumns)
+ optionsFinal.get("maxcolumns").get)
carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
GlobalDictionaryUtil.updateTableMetadataFunc = LoadTable.updateTableMetadata
val storePath = relation.tableMeta.storePath
@@ -561,7 +596,7 @@ case class LoadTable(
CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
}
if (carbonLoadModel.getLoadMetadataDetails.isEmpty && carbonLoadModel.getUseOnePass &&
- StringUtils.isEmpty(columnDict) && StringUtils.isEmpty(allDictionaryPath)) {
+ StringUtils.isEmpty(column_dict) && StringUtils.isEmpty(all_dictionary_path)) {
LOGGER.info(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
carbonLoadModel.setUseOnePass(false)
@@ -583,7 +618,7 @@ case class LoadTable(
.generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
dimensions, carbonLoadModel, sparkSession.sqlContext, storePath, dictFolderPath)
}
- if (!StringUtils.isEmpty(allDictionaryPath)) {
+ if (!StringUtils.isEmpty(all_dictionary_path)) {
carbonLoadModel.initPredefDictMap()
GlobalDictionaryUtil
.generateDictionaryFromDictionaryFiles(sparkSession.sqlContext,
@@ -592,7 +627,7 @@ case class LoadTable(
carbonTableIdentifier,
dictFolderPath,
dimensions,
- allDictionaryPath)
+ all_dictionary_path)
}
// dictionaryServerClient dictionary generator
val dictionaryServerPort = carbonProperty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a4083bf1/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 01e3ab6..429c5a3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -48,6 +48,7 @@ import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.NodeHolder;
@@ -201,6 +202,13 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
this.version = CarbonProperties.getInstance().getFormatVersion();
this.encoder = new TablePageEncoder(model);
+ String noInvertedIdxCol = "";
+ for (CarbonDimension cd : model.getSegmentProperties().getDimensions()) {
+ if (!cd.isUseInvertedIndex()) {
+ noInvertedIdxCol += (cd.getColName() + ",");
+ }
+ }
+ LOGGER.info("Columns considered as NoInverted Index are " + noInvertedIdxCol);
}
private void initParameters(CarbonFactDataHandlerModel model) {
[07/18] carbondata git commit: [CARBONDATA-1246] fix null pointer
exception by changing null to empty array. This closes #1113
Posted by ja...@apache.org.
[CARBONDATA-1246] fix null pointer exception by changing null to empty array. This closes #1113
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d9c3b483
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d9c3b483
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d9c3b483
Branch: refs/heads/encoding_override
Commit: d9c3b4837b81c02056a10446da18db0a1b427852
Parents: 49c64f7 659036f
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Wed Jul 5 11:23:46 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 11:23:46 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/datastore/filesystem/AlluxioCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/HDFSCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/LocalCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/ViewFSCarbonFile.java | 2 +-
.../core/datastore/filesystem/AlluxioCarbonFileTest.java | 3 ++-
.../carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java | 3 ++-
.../carbondata/core/datastore/filesystem/LocalCarbonFileTest.java | 3 ++-
.../core/datastore/filesystem/ViewFsCarbonFileTest.java | 3 ++-
8 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[08/18] carbondata git commit: modify compare test
Posted by ja...@apache.org.
modify compare test
fix
fix style
change table
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/327b307f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/327b307f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/327b307f
Branch: refs/heads/encoding_override
Commit: 327b307fdddc7b0fffe5b86049d1a2d08dfb182a
Parents: d9c3b48
Author: jackylk <ja...@huawei.com>
Authored: Mon Jul 3 21:54:39 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Wed Jul 5 21:34:56 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/CompareTest.scala | 103 ++++++++++++-------
1 file changed, 67 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/327b307f/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index ee53c31..ffc4b22 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -41,6 +41,7 @@ case class Query(sqlText: String, queryType: String, desc: String)
object CompareTest {
def parquetTableName: String = "comparetest_parquet"
+ def orcTableName: String = "comparetest_orc"
def carbonTableName(version: String): String = s"comparetest_carbonV$version"
// Table schema:
@@ -63,7 +64,7 @@ object CompareTest {
// +-------------+-----------+-------------+-------------+------------+
// | m4 | double | NA | measure | no |
// +-------------+-----------+-------------+-------------+------------+
- // | m5 | double | NA | measure | no |
+ // | m5 | decimal | NA | measure | no |
// +-------------+-----------+-------------+-------------+------------+
private def generateDataFrame(spark: SparkSession): DataFrame = {
val r = new Random()
@@ -71,10 +72,11 @@ object CompareTest {
.parallelize(1 to 10 * 1000 * 1000, 4)
.map { x =>
("city" + x % 8, "country" + x % 1103, "planet" + x % 10007, "IDENTIFIER" + x.toString,
- (x % 16).toShort, x / 2, (x << 1).toLong, x.toDouble / 13, x.toDouble / 11)
+ (x % 16).toShort, x / 2, (x << 1).toLong, x.toDouble / 13,
+ BigDecimal.valueOf(x.toDouble / 11))
}.map { x =>
- Row(x._1, x._2, x._3, x._4, x._5, x._6, x._7, x._8, x._9)
- }
+ Row(x._1, x._2, x._3, x._4, x._5, x._6, x._7, x._8, x._9)
+ }
val schema = StructType(
Seq(
@@ -86,7 +88,7 @@ object CompareTest {
StructField("m2", IntegerType, nullable = false),
StructField("m3", LongType, nullable = false),
StructField("m4", DoubleType, nullable = false),
- StructField("m5", DoubleType, nullable = false)
+ StructField("m5", DecimalType(30, 10), nullable = false)
)
)
@@ -142,12 +144,12 @@ object CompareTest {
// == FULL SCAN GROUP BY AGGREGATE ==
// ===========================================================================
Query(
- "select country, sum(m1) from $table group by country",
+ "select country, sum(m1) as metric from $table group by country order by metric",
"aggregate",
"group by on big data, on medium card column, medium result set,"
),
Query(
- "select city, sum(m1) from $table group by city",
+ "select city, sum(m1) as metric from $table group by city order by metric",
"aggregate",
"group by on big data, on low card column, small result set,"
),
@@ -170,17 +172,20 @@ object CompareTest {
// == FILTER SCAN GROUP BY AGGREGATION ==
// ===========================================================================
Query(
- "select country, sum(m1) from $table where city='city8' group by country ",
+ "select country, sum(m1) as metric from $table where city='city8' group by country " +
+ "order by metric",
"filter scan and aggregate",
"group by on large data, small result set"
),
Query(
- "select id, sum(m1) from $table where planet='planet10' group by id",
+ "select id, sum(m1) as metric from $table where planet='planet10' group by id " +
+ "order by metric",
"filter scan and aggregate",
"group by on medium data, large result set"
),
Query(
- "select city, sum(m1) from $table where country='country12' group by city ",
+ "select city, sum(m1) as metric from $table where country='country12' group by city " +
+ "order by metric",
"filter scan and aggregate",
"group by on medium data, small result set"
),
@@ -244,25 +249,35 @@ object CompareTest {
)
)
- private def loadParquetTable(spark: SparkSession, input: DataFrame): Double = time {
+ private def loadParquetTable(spark: SparkSession, input: DataFrame, table: String)
+ : Double = time {
// partitioned by last 1 digit of id column
val dfWithPartition = input.withColumn("partitionCol", input.col("id").%(10))
dfWithPartition.write
.partitionBy("partitionCol")
.mode(SaveMode.Overwrite)
- .parquet(parquetTableName)
+ .parquet(table)
+ spark.read.parquet(table).registerTempTable(table)
+ }
+
+ private def loadOrcTable(spark: SparkSession, input: DataFrame, table: String): Double = time {
+ // partitioned by last 1 digit of id column
+ input.write
+ .mode(SaveMode.Overwrite)
+ .orc(table)
+ spark.read.orc(table).registerTempTable(table)
}
- private def loadCarbonTable(spark: SparkSession, input: DataFrame, version: String): Double = {
+ private def loadCarbonTable(spark: SparkSession, input: DataFrame, tableName: String): Double = {
CarbonProperties.getInstance().addProperty(
CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
- version
+ "3"
)
- spark.sql(s"drop table if exists ${carbonTableName(version)}")
+ spark.sql(s"drop table if exists $tableName")
time {
input.write
.format("carbondata")
- .option("tableName", carbonTableName(version))
+ .option("tableName", tableName)
.option("tempCSV", "false")
.option("single_pass", "true")
.option("dictionary_exclude", "id") // id is high cardinality column
@@ -273,18 +288,23 @@ object CompareTest {
}
// load data into parquet, carbonV2, carbonV3
- private def prepareTable(spark: SparkSession): Unit = {
+ private def prepareTable(spark: SparkSession, table1: String, table2: String): Unit = {
val df = generateDataFrame(spark).cache
println(s"loading ${df.count} records, schema: ${df.schema}")
- val loadParquetTime = loadParquetTable(spark, df)
- val loadCarbonV3Time = loadCarbonTable(spark, df, version = "3")
- println(s"load completed, time: $loadParquetTime, $loadCarbonV3Time")
+ val table1Time = if (table1.endsWith("parquet")) {
+ loadParquetTable(spark, df, table1)
+ } else if (table1.endsWith("orc")) {
+ loadOrcTable(spark, df, table1)
+ } else {
+ sys.error("invalid table: " + table1)
+ }
+ val table2Time = loadCarbonTable(spark, df, table2)
+ println(s"load completed, time: $table1Time, $table2Time")
df.unpersist()
- spark.read.parquet(parquetTableName).registerTempTable(parquetTableName)
}
// Run all queries for the specified table
- private def runQueries(spark: SparkSession, tableName: String): Array[(Double, Int)] = {
+ private def runQueries(spark: SparkSession, tableName: String): Array[(Double, Array[Row])] = {
println(s"start running queries for $tableName...")
var result: Array[Row] = null
queries.zipWithIndex.map { case (query, index) =>
@@ -294,37 +314,46 @@ object CompareTest {
result = spark.sql(sqlText).collect()
}
println(s"=> $rt sec")
- (rt, result.length)
+ (rt, result)
+ }
+ }
+
+ private def printErrorIfNotMatch(index: Int, table1: String, result1: Array[Row],
+ table2: String, result2: Array[Row]): Unit = {
+ if (!result1.sameElements(result2)) {
+ val num = index + 1
+ println(s"$table1 result for query $num: ")
+ println(s"""${result1.mkString(",")}""")
+ println(s"$table2 result for query $num: ")
+ println(s"""${result2.mkString(",")}""")
+ sys.error(s"result not matching for query $num (${queries(index).desc})")
}
}
// run testcases and print comparison result
- private def runTest(spark: SparkSession): Unit = {
+ private def runTest(spark: SparkSession, table1: String, table2: String): Unit = {
val formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val date = new Date
val timestamp = date.getTime
// run queries on parquet and carbon
- val parquetResult: Array[(Double, Int)] = runQueries(spark, parquetTableName)
+ val table1Result: Array[(Double, Array[Row])] = runQueries(spark, table1)
// do GC and sleep for some time before running next table
System.gc()
Thread.sleep(1000)
System.gc()
Thread.sleep(1000)
- val carbonResult: Array[(Double, Int)] = runQueries(spark, carbonTableName("3"))
+ val table2Result: Array[(Double, Array[Row])] = runQueries(spark, table2)
// check result by comparing output from parquet and carbon
- parquetResult.zipWithIndex.foreach { case (result, index) =>
- if (result._2 != carbonResult(index)._2) {
- sys.error(s"result not matching for query ${index + 1} (${queries(index).desc}): " +
- s"${result._2} and ${carbonResult(index)._2}")
- }
+ table1Result.zipWithIndex.foreach { case (result, index) =>
+ printErrorIfNotMatch(index, table1, result._2, table2, table2Result(index)._2)
}
// print all response time in JSON format, so that it can be analyzed later
queries.zipWithIndex.foreach { case (query, index) =>
println("{" +
s""""query":"${index + 1}", """ +
- s""""parquetTime":${parquetResult(index)._1}, """ +
- s""""carbonTime":${carbonResult(index)._1}, """ +
- s""""fetched":${parquetResult(index)._2}, """ +
+ s""""$table1 time":${table1Result(index)._1}, """ +
+ s""""$table2 time":${table2Result(index)._1}, """ +
+ s""""fetched":${table1Result(index)._2.length}, """ +
s""""type":"${query.queryType}", """ +
s""""desc":"${query.desc}", """ +
s""""date": "${formatter.format(date)}" """ +
@@ -351,8 +380,10 @@ object CompareTest {
.getOrCreateCarbonSession(storeLocation)
spark.sparkContext.setLogLevel("warn")
- prepareTable(spark)
- runTest(spark)
+ val table1 = parquetTableName
+ val table2 = carbonTableName("3")
+ prepareTable(spark, table1, table2)
+ runTest(spark, table1, table2)
spark.close()
}
[02/18] carbondata git commit: [CARBONDATA-1241] Single sort to be
blocked for global sort. This closes #1109
Posted by ja...@apache.org.
[CARBONDATA-1241] Single sort to be blocked for global sort. This closes #1109
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1bd7b3da
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1bd7b3da
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1bd7b3da
Branch: refs/heads/encoding_override
Commit: 1bd7b3dafef440f13a187cd7f43a06eb05efff5d
Parents: b699ee6 a4083bf
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Wed Jul 5 09:12:59 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 09:12:59 2017 +0530
----------------------------------------------------------------------
.../execution/command/carbonTableSchema.scala | 1 +
.../execution/command/carbonTableSchema.scala | 221 +++++++++++--------
.../store/CarbonFactDataHandlerColumnar.java | 8 +
3 files changed, 137 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
[03/18] carbondata git commit: [CARBONDATA-1255] updated
ddl-operation-on-carbondata.md for column_group feature
Posted by ja...@apache.org.
[CARBONDATA-1255] updated ddl-operation-on-carbondata.md for column_group feature
This closes #1127
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/427b88b1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/427b88b1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/427b88b1
Branch: refs/heads/encoding_override
Commit: 427b88b1e48c8ca2ec1eb382b5617ee81040b437
Parents: 1bd7b3d
Author: vandana <va...@gmail.com>
Authored: Mon Jul 3 12:00:09 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Wed Jul 5 11:49:14 2017 +0800
----------------------------------------------------------------------
docs/ddl-operation-on-carbondata.md | 13 ++-----------
1 file changed, 2 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/427b88b1/docs/ddl-operation-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/ddl-operation-on-carbondata.md b/docs/ddl-operation-on-carbondata.md
index 66c9d30..79d1139 100644
--- a/docs/ddl-operation-on-carbondata.md
+++ b/docs/ddl-operation-on-carbondata.md
@@ -71,14 +71,7 @@ The following DDL operations are supported in CarbonData :
Here, DICTIONARY_EXCLUDE will exclude dictionary creation. This is applicable for high-cardinality columns and is an optional parameter. DICTIONARY_INCLUDE will generate dictionary for the columns specified in the list.
- - **Row/Column Format Configuration**
- Column groups with more than one column are stored in row format, instead of columnar format. By default, each column is a separate column group.
-
-```
- TBLPROPERTIES ('COLUMN_GROUPS'='(column1, column2),
- (Column3,Column4,Column5)')
-```
- **Table Block Size Configuration**
@@ -120,8 +113,7 @@ The following DDL operations are supported in CarbonData :
saleQuantity Int,
revenue Int)
STORED BY 'carbondata'
- TBLPROPERTIES ('COLUMN_GROUPS'='(productNumber,productName)',
- 'DICTIONARY_EXCLUDE'='storeCity',
+ TBLPROPERTIES ('DICTIONARY_EXCLUDE'='storeCity',
'DICTIONARY_INCLUDE'='productNumber',
'NO_INVERTED_INDEX'='productBatch')
```
@@ -402,8 +394,7 @@ of columns is used.
productBatch String,
revenue Int)
STORED BY 'carbondata'
- TBLPROPERTIES ('COLUMN_GROUPS'='(productNumber,saleQuantity)',
- 'DICTIONARY_EXCLUDE'='productName',
+ TBLPROPERTIES ('DICTIONARY_EXCLUDE'='productName',
'DICTIONARY_INCLUDE'='productNumber,saleQuantity',
'NO_INVERTED_INDEX'='productBatch',
'BUCKETNUMBER'='4',
[15/18] carbondata git commit: fix doc, remove invalid description
Posted by ja...@apache.org.
fix doc, remove invalid description
This closes #1151
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0558c286
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0558c286
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0558c286
Branch: refs/heads/encoding_override
Commit: 0558c286b3935d44a095f64677dc6abf276054fd
Parents: ff7eba0
Author: Liang Chen <ch...@apache.org>
Authored: Sun Jul 9 06:32:10 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Sun Jul 9 06:35:26 2017 +0800
----------------------------------------------------------------------
docs/configuration-parameters.md | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0558c286/docs/configuration-parameters.md
----------------------------------------------------------------------
diff --git a/docs/configuration-parameters.md b/docs/configuration-parameters.md
index deb5924..19f3f1a 100644
--- a/docs/configuration-parameters.md
+++ b/docs/configuration-parameters.md
@@ -144,6 +144,5 @@ This section provides the details of all the configurations required for CarbonD
|----------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| spark.driver.memory | 1g | Amount of memory to be used by the driver process. |
| spark.executor.memory | 1g | Amount of memory to be used per executor process. |
-| spark.sql.bigdata.register.analyseRule | org.apache.spark.sql.hive.acl.CarbonAccessControlRules | CarbonAccessControlRules need to be set for enabling Access Control. |
-
\ No newline at end of file
+
[18/18] carbondata git commit: [CARBONDATA-1229] acquired meta.lock
during table drop
Posted by ja...@apache.org.
[CARBONDATA-1229] acquired meta.lock during table drop
This closes #1153
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/403c3d9b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/403c3d9b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/403c3d9b
Branch: refs/heads/encoding_override
Commit: 403c3d9b41e166311ac45ec33b375cbecc8c4741
Parents: 619f1f9
Author: kunalkapoor <ku...@gmail.com>
Authored: Mon Jul 10 12:12:10 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Mon Jul 10 19:32:43 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/locks/CarbonLockUtil.java | 24 +++++++++
.../execution/command/carbonTableSchema.scala | 52 +++++++++-----------
.../org/apache/spark/util/AlterTableUtil.scala | 25 +---------
3 files changed, 50 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/403c3d9b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index fba03a1..eaaaf94 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.locks;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
/**
* This class contains all carbon lock utilities
@@ -60,4 +61,27 @@ public class CarbonLockUtil {
}
}
}
+
+ /**
+ * Given a lock type this method will return a new lock object if not acquired by any other
+ * operation
+ *
+ * @param carbonTable
+ * @param lockType
+ * @return
+ */
+ public static ICarbonLock getLockObject(CarbonTable carbonTable,
+ String lockType) {
+ ICarbonLock carbonLock = CarbonLockFactory
+ .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier().getCarbonTableIdentifier(),
+ lockType);
+ LOGGER.info("Trying to acquire lock: " + carbonLock);
+ if (carbonLock.lockWithRetries()) {
+ LOGGER.info("Successfully acquired the lock " + carbonLock);
+ } else {
+ throw new RuntimeException("Table is locked for updation. Please try after some time");
+ }
+ return carbonLock;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/403c3d9b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 8e7db45..2e5812c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -17,9 +17,8 @@
package org.apache.spark.sql.execution.command
-import java.io.File
-
import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
import scala.language.implicitConversions
import org.apache.commons.lang3.StringUtils
@@ -30,7 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.hive.{CarbonMetastore, CarbonRelation, HiveExternalCatalog}
+import org.apache.spark.sql.hive.{CarbonMetastore, CarbonRelation}
import org.apache.spark.util.FileUtils
import org.codehaus.jackson.map.ObjectMapper
@@ -41,10 +40,10 @@ import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
@@ -834,24 +833,17 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
val dbName = getDB.getDatabaseName(databaseNameOp, sparkSession)
val identifier = TableIdentifier(tableName, Option(dbName))
val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
- val carbonLock = CarbonLockFactory.getCarbonLockObj(carbonTableIdentifier.getDatabaseName,
- carbonTableIdentifier.getTableName + CarbonCommonConstants.UNDERSCORE +
- LockUsage.DROP_TABLE_LOCK)
+ val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.DROP_TABLE_LOCK)
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
val storePath = catalog.storePath
- var isLocked = false
catalog.checkSchemasModifiedTimeAndReloadTables()
+ val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
try {
- isLocked = carbonLock.lockWithRetries()
- if (isLocked) {
- logInfo("Successfully able to get the lock for drop.")
- }
- else {
- LOGGER.audit(s"Dropping table $dbName.$tableName failed as the Table is locked")
- sys.error("Table is locked for deletion. Please try after some time")
+ val carbonTable = catalog.getTableFromMetadata(dbName, tableName).map(_.carbonTable).orNull
+ locksToBeAcquired foreach {
+ lock => carbonLocks += CarbonLockUtil.getLockObject(carbonTable, lock)
}
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
- val carbonTable = catalog.getTableFromMetadata(dbName, tableName).map(_.carbonTable).orNull
if (null != carbonTable) {
// clear driver B-tree and dictionary cache
ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
@@ -859,18 +851,22 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
CarbonEnv.getInstance(sparkSession).carbonMetastore
.dropTable(storePath, identifier)(sparkSession)
LOGGER.audit(s"Deleted table [$tableName] under database [$dbName]")
+ } catch {
+ case ex: Exception =>
+ LOGGER.error(ex, s"Dropping table $dbName.$tableName failed")
} finally {
- if (carbonLock != null && isLocked) {
- if (carbonLock.unlock()) {
- logInfo("Table MetaData Unlocked Successfully after dropping the table")
- // deleting any remaining files.
- val metadataFilePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier).getMetadataDirectoryPath
- val fileType = FileFactory.getFileType(metadataFilePath)
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFiles(file.getParentFile)
- }
+ if (carbonLocks.nonEmpty) {
+ val unlocked = carbonLocks.forall(_.unlock())
+ if (unlocked) {
+ logInfo("Table MetaData Unlocked Successfully")
+ }
+ // deleting any remaining files.
+ val metadataFilePath = CarbonStorePath
+ .getCarbonTablePath(storePath, carbonTableIdentifier).getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataFilePath)
+ if (FileFactory.isFileExist(metadataFilePath, fileType)) {
+ val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
+ CarbonUtil.deleteFoldersAndFiles(file.getParentFile)
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/403c3d9b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 9e402cd..87717fb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.hive.HiveExternalCatalog._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock}
+import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock}
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.path.CarbonStorePath
@@ -65,7 +65,7 @@ object AlterTableUtil {
val acquiredLocks = ListBuffer[ICarbonLock]()
try {
locksToBeAcquired.foreach { lock =>
- acquiredLocks += getLockObject(table, lock)
+ acquiredLocks += CarbonLockUtil.getLockObject(table, lock)
}
acquiredLocks.toList
} catch {
@@ -76,27 +76,6 @@ object AlterTableUtil {
}
/**
- * Given a lock type this method will return a new lock object if not acquired by any other
- * operation
- *
- * @param carbonTable
- * @param lockType
- * @return
- */
- private def getLockObject(carbonTable: CarbonTable,
- lockType: String): ICarbonLock = {
- val carbonLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- lockType)
- if (carbonLock.lockWithRetries()) {
- LOGGER.info(s"Successfully acquired the lock $lockType")
- } else {
- sys.error("Table is locked for updation. Please try after some time")
- }
- carbonLock
- }
-
- /**
* This method will release the locks acquired for an operation
*
* @param locks
[17/18] carbondata git commit: [CARBONDATA-1282] Choose
BatchedDatasource scan only if schema fits codegen
Posted by ja...@apache.org.
[CARBONDATA-1282] Choose BatchedDatasource scan only if schema fits codegen
This closes #1148
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/619f1f95
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/619f1f95
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/619f1f95
Branch: refs/heads/encoding_override
Commit: 619f1f95429c71ec556b18104d3aff536ad2df89
Parents: c6bc1f0
Author: ashok.blend <as...@gmail.com>
Authored: Sat Jul 8 16:27:41 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Mon Jul 10 19:28:14 2017 +0530
----------------------------------------------------------------------
.../apache/spark/sql/execution/CarbonLateDecodeStrategy.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/619f1f95/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
index 4605914..a206bef 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
@@ -524,7 +524,9 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT)
}
}
- sqlContext.conf.wholeStageEnabled && vectorizedReader.toBoolean &&
+ val supportCodegen =
+ sqlContext.conf.wholeStageEnabled && sqlContext.conf.wholeStageMaxNumFields >= cols.size
+ supportCodegen && vectorizedReader.toBoolean &&
cols.forall(_.dataType.isInstanceOf[AtomicType])
}
}
[04/18] carbondata git commit: performance issue resolved
Posted by ja...@apache.org.
performance issue resolved
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1278c41b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1278c41b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1278c41b
Branch: refs/heads/encoding_override
Commit: 1278c41bb1692a3e542194626346c3324a5cfb4e
Parents: 427b88b
Author: sahakushal <ca...@gmail.com>
Authored: Wed Jun 28 16:29:01 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 10:38:56 2017 +0530
----------------------------------------------------------------------
.../apache/carbondata/spark/load/CarbonLoaderUtil.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1278c41b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 54e12f3..5b603aa 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -742,14 +742,20 @@ public final class CarbonLoaderUtil {
if (null == nodeAndBlockMapping.get(node)) {
list = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
list.add(nbr.getBlock());
- Collections.sort(list);
nodeAndBlockMapping.put(node, list);
} else {
list = nodeAndBlockMapping.get(node);
list.add(nbr.getBlock());
- Collections.sort(list);
}
}
+ /*for resolving performance issue, removed values() with entrySet () iterating the values and
+ sorting it.entrySet will give the logical view for hashMap and we dont query the map twice for
+ each key whereas values () iterate twice*/
+ Iterator<Map.Entry<String, List<Distributable>>> iterator =
+ nodeAndBlockMapping.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Collections.sort(iterator.next().getValue());
+ }
}
/**
[14/18] carbondata git commit: [CARBONDATA-1280] Solve HiveExample
dependency issues and fix spark 1.6 CI
Posted by ja...@apache.org.
[CARBONDATA-1280] Solve HiveExample dependency issues and fix spark 1.6 CI
This closes #1150
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ff7eba0b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ff7eba0b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ff7eba0b
Branch: refs/heads/encoding_override
Commit: ff7eba0b838bbf4b9361d5a6a0859d7af104b9a5
Parents: c7e7f70
Author: chenliang613 <ch...@apache.org>
Authored: Sat Jul 8 23:53:02 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Sat Jul 8 23:58:57 2017 +0800
----------------------------------------------------------------------
integration/hive/pom.xml | 18 ++++++++++++++++++
pom.xml | 6 ++++--
2 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff7eba0b/integration/hive/pom.xml
----------------------------------------------------------------------
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 5a33958..3f1d6ef 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -78,6 +78,24 @@
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive-thriftserver_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-repl_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.10</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff7eba0b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7065bee..7af7b64 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,8 +103,6 @@
<module>integration/spark-common-test</module>
<module>assembly</module>
<module>examples/flink</module>
- <module>integration/hive</module>
- <module>integration/presto</module>
</modules>
@@ -312,6 +310,8 @@
<module>examples/spark</module>
<module>integration/spark2</module>
<module>examples/spark2</module>
+ <module>integration/hive</module>
+ <module>integration/presto</module>
</modules>
</profile>
<profile>
@@ -363,6 +363,8 @@
</properties>
<modules>
<module>integration/spark2</module>
+ <module>integration/hive</module>
+ <module>integration/presto</module>
<module>examples/spark2</module>
</modules>
</profile>
[16/18] carbondata git commit: [CARBONDATA-1267] Add short_int case
branch in DeltaIntegalCodec
Posted by ja...@apache.org.
[CARBONDATA-1267] Add short_int case branch in DeltaIntegalCodec
This closes #1139
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c6bc1f07
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c6bc1f07
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c6bc1f07
Branch: refs/heads/encoding_override
Commit: c6bc1f07180c4740ba9b3e518ead34daa093be1c
Parents: 0558c28
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jul 5 21:00:45 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Jul 10 21:19:45 2017 +0800
----------------------------------------------------------------------
.../page/encoding/DeltaIntegerCodec.java | 12 +++++++
.../test/resources/short_int_as_target_type.csv | 3 ++
.../aggquery/IntegerDataTypeTestCase.scala | 33 +++++++++++++++++++-
3 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c6bc1f07/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
index b77f7a2..6cf59a6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
@@ -120,6 +120,9 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
case SHORT:
encodedPage.putShort(rowId, (short)(max - value));
break;
+ case SHORT_INT:
+ encodedPage.putShortInt(rowId, (int)(max - value));
+ break;
case INT:
encodedPage.putInt(rowId, (int)(max - value));
break;
@@ -137,6 +140,9 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
case SHORT:
encodedPage.putShort(rowId, (short)(max - value));
break;
+ case SHORT_INT:
+ encodedPage.putShortInt(rowId, (int)(max - value));
+ break;
case INT:
encodedPage.putInt(rowId, (int)(max - value));
break;
@@ -157,6 +163,9 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
case SHORT:
encodedPage.putShort(rowId, (short)(max - value));
break;
+ case SHORT_INT:
+ encodedPage.putShortInt(rowId, (int)(max - value));
+ break;
case INT:
encodedPage.putInt(rowId, (int)(max - value));
break;
@@ -177,6 +186,9 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
case SHORT:
encodedPage.putShort(rowId, (short)(max - value));
break;
+ case SHORT_INT:
+ encodedPage.putShortInt(rowId, (int)(max - value));
+ break;
case INT:
encodedPage.putInt(rowId, (int)(max - value));
break;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c6bc1f07/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv b/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
new file mode 100644
index 0000000..964517e
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/short_int_as_target_type.csv
@@ -0,0 +1,3 @@
+begin_time,name
+1497376581,name1
+1497423838,name2
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c6bc1f07/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
index 8e19684..f918301 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/aggquery/IntegerDataTypeTestCase.scala
@@ -112,6 +112,37 @@ class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
}
+ test("short int as target type in deltaIntegerCodec") {
+ sql(
+ """
+ | DROP TABLE IF EXISTS short_int_target_table
+ """.stripMargin)
+
+ //begin_time column will be encoded by deltaIntegerCodec
+ sql(
+ """
+ | CREATE TABLE short_int_target_table
+ | (begin_time bigint, name string)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$resourcesPath/short_int_as_target_type.csv'
+ | INTO TABLE short_int_target_table
+ """.stripMargin)
+
+ checkAnswer(
+ sql("select begin_time from short_int_target_table"),
+ Seq(Row(1497376581), Row(1497423838))
+ )
+
+ sql(
+ """
+ | DROP TABLE short_int_target_table
+ """.stripMargin)
+ }
+
override def afterAll {
sql("drop table if exists integertypetableAgg")
CarbonProperties.getInstance().addProperty(
@@ -119,4 +150,4 @@ class IntegerDataTypeTestCase extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING_DEFAULT
)
}
-}
\ No newline at end of file
+}
[10/18] carbondata git commit: [CARBONDATA-1214] Changing the delete
syntax as in the hive for segment deletion
Posted by ja...@apache.org.
[CARBONDATA-1214] Changing the delete syntax as in the hive for segment deletion
This closes #1078
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/05de7fda
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/05de7fda
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/05de7fda
Branch: refs/heads/encoding_override
Commit: 05de7fdae5f02f172321af614532bbc331309fcb
Parents: 26d2f1c
Author: ravikiran23 <ra...@gmail.com>
Authored: Thu Jun 22 18:18:19 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Jul 6 16:03:29 2017 +0530
----------------------------------------------------------------------
.../examples/DataManagementExample.scala | 4 ++--
.../MajorCompactionIgnoreInMinorTest.scala | 4 ++--
.../dataload/TestLoadTblNameIsKeyword.scala | 2 +-
.../DataRetentionConcurrencyTestCase.scala | 4 ++--
.../dataretention/DataRetentionTestCase.scala | 24 ++++++++++----------
.../iud/HorizontalCompactionTestCase.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 1 +
.../org/apache/spark/sql/CarbonSqlParser.scala | 21 ++++++++---------
.../DataCompactionNoDictionaryTest.scala | 2 +-
.../sql/parser/CarbonSpark2SqlParser.scala | 18 +++++++--------
10 files changed, 39 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
index 4552e06..551a008 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/DataManagementExample.scala
@@ -45,7 +45,7 @@ object DataManagementExample {
cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
// delete the first segment
- cc.sql("DELETE SEGMENT 0 FROM TABLE t3")
+ cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.ID IN (0)")
cc.sql("SHOW SEGMENTS FOR TABLE t3 LIMIT 10").show
// this query will be executed on last 4 segments, it should return 4000 rows
@@ -63,7 +63,7 @@ object DataManagementExample {
cc.sql("SELECT count(*) AS amount FROM t3").show
// delete all segments whose loading time is before '2099-01-01 01:00:00'
- cc.sql("DELETE SEGMENTS FROM TABLE t3 WHERE STARTTIME BEFORE '2099-01-01 01:00:00'")
+ cc.sql("DELETE FROM TABLE T3 WHERE SEGMENT.STARTTIME BEFORE '2099-01-01 01:00:00'")
cc.sql("SHOW SEGMENTS FOR TABLE t3 ").show
// this query will be executed on 0 segments, it should return 0 rows
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 9d2cf96..b66e37b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -103,7 +103,7 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
*/
test("delete compacted segment and check status") {
try {
- sql("delete segment 2 from table ignoremajor")
+ sql("delete from table ignoremajor where segment.id in (2)")
assert(false)
}
catch {
@@ -128,7 +128,7 @@ class MajorCompactionIgnoreInMinorTest extends QueryTest with BeforeAndAfterAll
*/
test("delete compacted segment by date and check status") {
sql(
- "DELETE SEGMENTS FROM TABLE ignoremajor where STARTTIME before" +
+ "delete from table ignoremajor where segment.starttime before " +
" '2222-01-01 19:35:01'"
)
val carbontablePath = CarbonStorePath
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
index cadaa6e..71aeb99 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
@@ -78,7 +78,7 @@ class TestLoadTblNameIsKeyword extends QueryTest with BeforeAndAfterAll {
LOAD DATA LOCAL INPATH '$testData' into table timestamp
""")
sql("show segments for table timestamp")
- sql("delete segments from table timestamp where starttime before '2099-10-01 18:00:00'")
+ sql("delete from table timestamp where segment.starttime before '2099-10-01 18:00:00'")
sql("clean files for table timestamp")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
index 784382b..79350eb 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionConcurrencyTestCase.scala
@@ -57,7 +57,7 @@ class DataRetentionConcurrencyTestCase extends QueryTest with BeforeAndAfterAll
val tasks = new util.ArrayList[Callable[String]]()
tasks
.add(new QueryTask(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE concurrent OPTIONS('DELIMITER' = ',')"))
- tasks.add(new QueryTask("Delete segment 0 from table concurrent"))
+ tasks.add(new QueryTask("delete from table concurrent where segment.id in (0)"))
tasks.add(new QueryTask("clean files for table concurrent"))
val results = executorService.invokeAll(tasks)
for (i <- 0 until tasks.size()) {
@@ -77,7 +77,7 @@ class DataRetentionConcurrencyTestCase extends QueryTest with BeforeAndAfterAll
.add(new QueryTask(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE concurrent OPTIONS('DELIMITER' = ',')"))
tasks
.add(new QueryTask(
- "DELETE SEGMENTS FROM TABLE concurrent where STARTTIME before '2099-01-01 00:00:00'"))
+ "delete from table concurrent where segment.starttime before '2099-01-01 00:00:00'"))
tasks.add(new QueryTask("clean files for table concurrent"))
val results = executorService.invokeAll(tasks)
for (i <- 0 until tasks.size()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index 684ed8e..b255099 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -131,7 +131,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
val actualValue: String = getSegmentStartTime(segments, 1)
// delete segments (0,1) which contains ind, aus
sql(
- "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before '" + actualValue + "'")
+ "delete from table DataRetentionTable where segment.starttime before '" + actualValue + "'")
// load segment 2 which contains eng
sql(
@@ -147,7 +147,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
test("RetentionTest3_DeleteByLoadId") {
// delete segment 2 and load ind segment
- sql("DELETE SEGMENT 2 FROM TABLE DataRetentionTable")
+ sql("delete from table DataRetentionTable where segment.id in (2)")
sql(
s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE DataRetentionTable " +
"OPTIONS('DELIMITER' = ',')")
@@ -166,7 +166,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
test("RetentionTest4_DeleteByInvalidLoadId") {
try {
// delete segment with no id
- sql("DELETE SEGMENT FROM TABLE DataRetentionTable")
+ sql("delete from table DataRetentionTable where segment.id in ()")
assert(false)
} catch {
case e: MalformedCarbonCommandException =>
@@ -191,8 +191,8 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("select count(*) from carbon_table_1"), Seq(Row(20)))
- sql("delete segments from table carbon_table_1 " +
- "where starttime before '2099-07-28 11:00:00'")
+ sql("delete from table carbon_table_1 where segment.starttime " +
+ " before '2099-07-28 11:00:00'")
checkAnswer(
sql("select count(*) from carbon_table_1"), Seq(Row(0)))
@@ -204,7 +204,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
try {
sql(
- "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before" +
+ "delete from table DataRetentionTable where segment.starttime before" +
" 'abcd-01-01 00:00:00'")
assert(false)
} catch {
@@ -215,7 +215,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
try {
sql(
- "DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before" +
+ "delete from table DataRetentionTable where segment.starttime before" +
" '2099:01:01 00:00:00'")
assert(false)
} catch {
@@ -230,7 +230,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
),
Seq(Row("ind", 9))
)
- sql("DELETE SEGMENTS FROM TABLE DataRetentionTable where STARTTIME before '2099-01-01'")
+ sql("delete from table DataRetentionTable where segment.starttime before '2099-01-01'")
checkAnswer(
sql("SELECT country, count(salary) AS amount FROM DataRetentionTable WHERE country" +
" IN ('china','ind','aus','eng') GROUP BY country"), Seq())
@@ -280,7 +280,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
// delete segment 0 it should fail
try {
- sql("DELETE SEGMENT 0 FROM TABLE retentionlock")
+ sql("delete from table retentionlock where segment.id in (0)")
throw new MalformedCarbonCommandException("Invalid")
} catch {
case me: MalformedCarbonCommandException =>
@@ -291,7 +291,7 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
// it should fail
try {
- sql("DELETE SEGMENTS FROM TABLE retentionlock where STARTTIME before " +
+ sql("delete from table retentionlock where segment.starttime before " +
"'2099-01-01 00:00:00.0'")
throw new MalformedCarbonCommandException("Invalid")
} catch {
@@ -317,10 +317,10 @@ class DataRetentionTestCase extends QueryTest with BeforeAndAfterAll {
carbonCleanFilesLock.unlock()
carbonDeleteSegmentLock.unlock()
- sql("DELETE SEGMENT 0 FROM TABLE retentionlock")
+ sql("delete from table retentionlock where segment.id in (0)")
//load and delete should execute parallely
carbonMetadataLock.lockWithRetries()
- sql("DELETE SEGMENT 1 FROM TABLE retentionlock")
+ sql("delete from table retentionlock where segment.id in (1)")
carbonMetadataLock.unlock()
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
index d8310da..0d30333 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/HorizontalCompactionTestCase.scala
@@ -321,7 +321,7 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql(
"""delete from dest2 where (c2 < 3) or (c2 > 10 and c2 < 13) or (c2 > 20 and c2 < 23) or (c2 > 30 and c2 < 33)""").show()
- sql("""DELETE SEGMENT 0 FROM TABLE dest2""")
+ sql("""delete from table dest2 where segment.id in (0) """)
sql("""clean files for table dest2""")
sql(
"""update dest2 set (c5) = ('8RAM size') where (c2 > 3 and c2 < 5) or (c2 > 13 and c2 < 15) or (c2 > 23 and c2 < 25) or (c2 > 33 and c2 < 35)""")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 8207a9d..4dbdc8d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -164,6 +164,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val SMALLINT = carbonKeyWord("SMALLINT")
protected val CHANGE = carbonKeyWord("CHANGE")
protected val TBLPROPERTIES = carbonKeyWord("TBLPROPERTIES")
+ protected val ID = carbonKeyWord("ID")
protected val doubleQuotedString = "\"([^\"]+)\"".r
protected val singleQuotedString = "'([^']+)'".r
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index a664104..9dc9ee2 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -417,23 +417,20 @@ class CarbonSqlParser() extends CarbonDDLSqlParser {
}
protected lazy val deleteLoadsByID: Parser[LogicalPlan] =
- DELETE ~> SEGMENT ~> repsep(segmentId, ",") ~ (FROM ~> TABLE ~>
- (ident <~ ".").? ~ ident) <~
- opt(";") ^^ {
- case loadids ~ table => table match {
- case databaseName ~ tableName =>
- DeleteLoadsById(loadids, convertDbNameToLowerCase(databaseName), tableName.toLowerCase())
- }
- }
+ DELETE ~> FROM ~ TABLE ~> (ident <~ ".").? ~ ident ~
+ (WHERE ~> (SEGMENT ~ "." ~ ID) ~> IN ~> "(" ~> repsep(segmentId, ",")) <~ ")" ~ opt(";") ^^ {
+ case dbName ~ tableName ~ loadids =>
+ DeleteLoadsById(loadids, convertDbNameToLowerCase(dbName), tableName.toLowerCase())
+ }
protected lazy val deleteLoadsByLoadDate: Parser[LogicalPlan] =
- DELETE ~> SEGMENTS ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
- (WHERE ~> (STARTTIME <~ BEFORE) ~ stringLit) <~
+ DELETE ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
+ (WHERE ~> (SEGMENT ~ "." ~ STARTTIME ~> BEFORE) ~ stringLit) <~
opt(";") ^^ {
- case schema ~ table ~ condition =>
+ case database ~ table ~ condition =>
condition match {
case dateField ~ dateValue =>
- DeleteLoadsByLoadDate(convertDbNameToLowerCase(schema),
+ DeleteLoadsByLoadDate(convertDbNameToLowerCase(database),
table.toLowerCase(),
dateField,
dateValue)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
index dda2a88..5897681 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
@@ -156,7 +156,7 @@ class DataCompactionNoDictionaryTest extends QueryTest with BeforeAndAfterAll {
sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE nodictionaryCompaction " +
"OPTIONS('DELIMITER' = ',')"
)
- sql("DELETE segment 0.1,3 FROM TABLE nodictionaryCompaction")
+ sql("delete from table nodictionaryCompaction where segment.id in (0.1,3)")
checkAnswer(
sql("select country from nodictionaryCompaction"),
Seq()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05de7fda/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index af286eb..511a61c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -220,23 +220,21 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser {
}
protected lazy val deleteLoadsByID: Parser[LogicalPlan] =
- DELETE ~> SEGMENT ~> repsep(segmentId, ",") ~ (FROM ~> TABLE ~>
- (ident <~ ".").? ~ ident) <~
+ DELETE ~> FROM ~ TABLE ~> (ident <~ ".").? ~ ident ~
+ (WHERE ~> (SEGMENT ~ "." ~ ID) ~> IN ~> "(" ~> repsep(segmentId, ",")) <~ ")" ~
opt(";") ^^ {
- case loadids ~ table => table match {
- case databaseName ~ tableName =>
- DeleteLoadsById(loadids, convertDbNameToLowerCase(databaseName), tableName.toLowerCase())
- }
+ case dbName ~ tableName ~ loadids =>
+ DeleteLoadsById(loadids, dbName, tableName.toLowerCase())
}
protected lazy val deleteLoadsByLoadDate: Parser[LogicalPlan] =
- DELETE ~> SEGMENTS ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
- (WHERE ~> (STARTTIME <~ BEFORE) ~ stringLit) <~
+ DELETE ~> FROM ~> TABLE ~> (ident <~ ".").? ~ ident ~
+ (WHERE ~> (SEGMENT ~ "." ~ STARTTIME ~> BEFORE) ~ stringLit) <~
opt(";") ^^ {
- case schema ~ table ~ condition =>
+ case database ~ table ~ condition =>
condition match {
case dateField ~ dateValue =>
- DeleteLoadsByLoadDate(convertDbNameToLowerCase(schema),
+ DeleteLoadsByLoadDate(convertDbNameToLowerCase(database),
table.toLowerCase(),
dateField,
dateValue)
[09/18] carbondata git commit: [CARBONDATA-1259] CompareTest
improvement This closes #1129
Posted by ja...@apache.org.
[CARBONDATA-1259] CompareTest improvement This closes #1129
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/26d2f1c8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/26d2f1c8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/26d2f1c8
Branch: refs/heads/encoding_override
Commit: 26d2f1c83c64a677221012319d1ec86aee429103
Parents: d9c3b48 327b307
Author: chenliang613 <ch...@apache.org>
Authored: Wed Jul 5 21:35:45 2017 +0800
Committer: chenliang613 <ch...@apache.org>
Committed: Wed Jul 5 21:35:45 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/CompareTest.scala | 103 ++++++++++++-------
1 file changed, 67 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
[05/18] carbondata git commit: [CARBONDATA-1242] Block distribution
performance improvement. This closes #1108
Posted by ja...@apache.org.
[CARBONDATA-1242] Block distribution performance improvement. This closes #1108
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/49c64f71
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/49c64f71
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/49c64f71
Branch: refs/heads/encoding_override
Commit: 49c64f713f9a12a627de0d40ee238fdc3660189e
Parents: 1bd7b3d 1278c41
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Wed Jul 5 10:39:54 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 10:39:54 2017 +0530
----------------------------------------------------------------------
docs/ddl-operation-on-carbondata.md | 13 ++-----------
.../apache/carbondata/spark/load/CarbonLoaderUtil.java | 10 ++++++++--
2 files changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[11/18] carbondata git commit: [CARBONDATA-1270] Documentation update
for Delete by ID and DATE syntax and example
Posted by ja...@apache.org.
[CARBONDATA-1270] Documentation update for Delete by ID and DATE syntax and example
This closes #1141
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5af529e0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5af529e0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5af529e0
Branch: refs/heads/encoding_override
Commit: 5af529e0e0ab54fb592774bd4f6eae11d5000187
Parents: 05de7fd
Author: sgururajshetty <sg...@gmail.com>
Authored: Thu Jul 6 11:53:38 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Jul 6 16:22:33 2017 +0530
----------------------------------------------------------------------
docs/dml-operation-on-carbondata.md | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5af529e0/docs/dml-operation-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/dml-operation-on-carbondata.md b/docs/dml-operation-on-carbondata.md
index 45f7464..4928098 100644
--- a/docs/dml-operation-on-carbondata.md
+++ b/docs/dml-operation-on-carbondata.md
@@ -262,14 +262,13 @@ Using this segment ID, you can remove the segment.
The following command will get the segmentID.
```
-SHOW SEGMENTS FOR Table dbname.tablename LIMIT number_of_segments
+SHOW SEGMENTS FOR Table [db_name.]table_name LIMIT number_of_segments
```
After you retrieve the segment ID of the segment that you want to delete, execute the following command to delete the selected segment.
```
-DELETE SEGMENT segment_sequence_id1, segments_sequence_id2, ....
-FROM TABLE tableName
+DELETE FROM TABLE [db_name.]table_name WERE SEGMENT.ID IN (segment_id1, segments_id2, ....)
```
### Parameter Description
@@ -282,8 +281,8 @@ FROM TABLE tableName
### Example:
```
-DELETE SEGMENT 0 FROM TABLE CarbonDatabase.CarbonTable;
-DELETE SEGMENT 0.1,5,8 FROM TABLE CarbonDatabase.CarbonTable;
+DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.ID IN (0);
+DELETE FROM TABLE CarbonDatabase.CarbonTable WHERE SEGMENT.ID IN (0,5,8);
```
NOTE: Here 0.1 is compacted segment sequence id.
@@ -293,8 +292,8 @@ This command will allow to delete the CarbonData segment(s) from the store based
The segment created before the particular date will be removed from the specific stores.
```
-DELETE SEGMENTS FROM TABLE [db_name.]table_name
-WHERE STARTTIME BEFORE DATE_VALUE
+DELETE FROM TABLE [db_name.]table_name
+WHERE SEGMENT.STARTTIME BEFORE DATE_VALUE
```
### Parameter Description
@@ -308,8 +307,8 @@ WHERE STARTTIME BEFORE DATE_VALUE
### Example:
```
- DELETE SEGMENTS FROM TABLE CarbonDatabase.CarbonTable
- WHERE STARTTIME BEFORE '2017-06-01 12:05:06';
+ DELETE FROM TABLE CarbonDatabase.CarbonTable
+ WHERE SEGMENT.STARTTIME BEFORE '2017-06-01 12:05:06';
```
## Update CarbonData Table
[12/18] carbondata git commit: [CARBONDATA-1266][PRESTO] Fixed issue
for non existing table
Posted by ja...@apache.org.
[CARBONDATA-1266][PRESTO] Fixed issue for non existing table
This closes #1137
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/92d1d970
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/92d1d970
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/92d1d970
Branch: refs/heads/encoding_override
Commit: 92d1d970a24f4dbd975316a35cf9b38a3ccdb111
Parents: 5af529e
Author: jatin <ja...@knoldus.in>
Authored: Wed Jul 5 17:34:19 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jul 7 08:37:19 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/carbondata/presto/CarbondataMetadata.java | 6 ------
.../org/apache/carbondata/presto/impl/CarbonTableReader.java | 3 ++-
2 files changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d1d970/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 7701490..f2d19cf 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -115,9 +115,6 @@ public class CarbondataMetadata implements ConnectorMetadata {
}
CarbonTable carbonTable = carbonTableReader.getTable(schemaTableName);
- if (carbonTable == null) {
- return null;
- }
List<ColumnMetadata> columnsMetaList = new LinkedList<>();
List<CarbonColumn> carbonColumns = carbonTable.getCreateOrderColumn(schemaTableName.getTableName());
@@ -148,9 +145,6 @@ public class CarbondataMetadata implements ConnectorMetadata {
//CarbonTable(official struct) is stored in CarbonMetadata(official struct)
CarbonTable cb = carbonTableReader.getTable(handle.getSchemaTableName());
- if (cb == null) {
- throw new TableNotFoundException(handle.getSchemaTableName());
- }
ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
String tableName = handle.getSchemaTableName().getTableName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d1d970/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index 54832f5..e4d3ba5 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -65,6 +65,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.util.Objects.requireNonNull;
+import com.facebook.presto.spi.TableNotFoundException;
/** CarbonTableReader will be a facade of these utils
*
@@ -251,7 +252,7 @@ public class CarbonTableReader {
return parseCarbonMetadata(table);
}
- return null;
+ throw new TableNotFoundException(schemaTableName);
}
/**
[06/18] carbondata git commit: fix null pointer exception by changing
null to empty array
Posted by ja...@apache.org.
fix null pointer exception by changing null to empty array
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/659036fe
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/659036fe
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/659036fe
Branch: refs/heads/encoding_override
Commit: 659036fee8dd1645eea31eeab0423bd0d5c03f19
Parents: 49c64f7
Author: Jin Guodong <je...@gmail.com>
Authored: Thu Jun 29 12:43:51 2017 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Wed Jul 5 11:21:47 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/datastore/filesystem/AlluxioCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/HDFSCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/LocalCarbonFile.java | 2 +-
.../carbondata/core/datastore/filesystem/ViewFSCarbonFile.java | 2 +-
.../core/datastore/filesystem/AlluxioCarbonFileTest.java | 3 ++-
.../carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java | 3 ++-
.../carbondata/core/datastore/filesystem/LocalCarbonFileTest.java | 3 ++-
.../core/datastore/filesystem/ViewFsCarbonFileTest.java | 3 ++-
8 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
index 36e4868..c3ccd0c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
@@ -74,7 +74,7 @@ public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
Path path = fileStatus.getPath();
listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
} else {
- return null;
+ return new CarbonFile[0];
}
} catch (IOException e) {
LOGGER.error("Exception occured: " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index 75d9fc3..19bdc7b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -72,7 +72,7 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
Path path = fileStatus.getPath();
listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
} else {
- return null;
+ return new CarbonFile[0];
}
} catch (IOException e) {
LOGGER.error("Exception occured: " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 9f90713..89a5cd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -123,7 +123,7 @@ public class LocalCarbonFile implements CarbonFile {
@Override public CarbonFile[] listFiles() {
if (!file.isDirectory()) {
- return null;
+ return new CarbonFile[0];
}
File[] files = file.listFiles();
if (files == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
index da15371..e05112d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
@@ -71,7 +71,7 @@ public class ViewFSCarbonFile extends AbstractDFSCarbonFile {
Path path = fileStatus.getPath();
listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
} else {
- return null;
+ return new CarbonFile[0];
}
} catch (IOException ex) {
LOGGER.error("Exception occured" + ex.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
index eff0d8b..f55cd67 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
@@ -35,6 +35,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -90,7 +91,7 @@ public class AlluxioCarbonFileTest {
@Test
public void testListFilesWithOutDirectoryPermission() {
alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(alluxioCarbonFile.listFiles() == null);
+ assertArrayEquals(alluxioCarbonFile.listFiles(), new CarbonFile[0]);
}
@Test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index eeecb03..7726693 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -39,6 +39,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -132,7 +133,7 @@ public class HDFSCarbonFileTest {
}
};
- assertEquals(hdfsCarbonFile.listFiles(), null);
+ assertArrayEquals(hdfsCarbonFile.listFiles(), new CarbonFile[0]);
}
@Test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 3925e9e..9ae5c34 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -30,6 +30,7 @@ import java.io.*;
import java.nio.channels.ReadableByteChannel;
import java.util.Objects;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -102,7 +103,7 @@ public class LocalCarbonFileTest {
return false;
}
};
- assertTrue(localCarbonFile.listFiles() == null);
+ assertArrayEquals(localCarbonFile.listFiles(), new CarbonFile[0]);
}
@Test
http://git-wip-us.apache.org/repos/asf/carbondata/blob/659036fe/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
index 3cb8198..ba661b1 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -35,6 +35,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -91,7 +92,7 @@ public class ViewFsCarbonFileTest {
@Test
public void testListFilesWithOutDirectoryPermission() {
viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(viewFSCarbonFile.listFiles() == null);
+ assertArrayEquals(viewFSCarbonFile.listFiles(), new CarbonFile[0]);
}
@Test
[13/18] carbondata git commit: [CARBONDATA-1269][PRESTO] Fixed bug
for select operation in non existing database
Posted by ja...@apache.org.
[CARBONDATA-1269][PRESTO] Fixed bug for select operation in non existing database
This closes #1143
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c7e7f70b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c7e7f70b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c7e7f70b
Branch: refs/heads/encoding_override
Commit: c7e7f70b827aba3fbd3aab849cc2a50257627cd5
Parents: 92d1d97
Author: Geetika Gupta <ge...@knoldus.in>
Authored: Thu Jul 6 11:44:06 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jul 7 08:41:16 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/carbondata/presto/CarbondataMetadata.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c7e7f70b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index f2d19cf..a132fb4 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -111,7 +111,7 @@ public class CarbondataMetadata implements ConnectorMetadata {
private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) {
if (!listSchemaNamesInternal().contains(schemaTableName.getSchemaName())) {
- return null;
+ throw new SchemaNotFoundException(schemaTableName.getSchemaName());
}
CarbonTable carbonTable = carbonTableReader.getTable(schemaTableName);