You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/20 07:29:30 UTC
[24/56] [abbrv] carbondata git commit: added condition to skip
single_pass for first load
added condition to skip single_pass for first load
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2aa5d83b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2aa5d83b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2aa5d83b
Branch: refs/heads/streaming_ingest
Commit: 2aa5d83bb2f0c5eb2a74fff9e73d2e9e3b5482bc
Parents: 4319be8
Author: kunal642 <ku...@knoldus.in>
Authored: Wed Jun 14 13:21:47 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Jun 15 14:08:48 2017 +0530
----------------------------------------------------------------------
.../carbondata/spark/rdd/CarbonDataRDDFactory.scala | 4 ----
.../spark/sql/execution/command/carbonTableSchema.scala | 9 +++++++++
.../carbondata/spark/rdd/CarbonDataRDDFactory.scala | 4 ----
.../spark/sql/execution/command/carbonTableSchema.scala | 12 ++++++++++--
4 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2aa5d83b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index ee999a4..de0fd5a 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -444,10 +444,6 @@ object CarbonDataRDDFactory {
// Check if any load need to be deleted before loading new data
DataManagementFunc.deleteLoadsAndUpdateMetadata(carbonLoadModel.getDatabaseName,
carbonLoadModel.getTableName, storePath, false, carbonTable)
- if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
- }
-
var currentLoadCount = -1
val convLoadDetails = carbonLoadModel.getLoadMetadataDetails.asScala
// taking the latest segment ID present.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2aa5d83b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 426d340..d085ad7 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -496,6 +496,15 @@ case class LoadTable(
val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
val dimensions = carbonTable.getDimensionByTableName(
carbonTable.getFactTableName).asScala.toArray
+ if (null == carbonLoadModel.getLoadMetadataDetails) {
+ CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
+ }
+ if (carbonLoadModel.getLoadMetadataDetails.isEmpty && carbonLoadModel.getUseOnePass &&
+ StringUtils.isEmpty(columnDict) && StringUtils.isEmpty(allDictionaryPath)) {
+ LOGGER.info(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
+ LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
+ carbonLoadModel.setUseOnePass(false)
+ }
if (carbonLoadModel.getUseOnePass) {
val colDictFilePath = carbonLoadModel.getColDictFilePath
if (!StringUtils.isEmpty(colDictFilePath)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2aa5d83b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 93d790b..8871f3b 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -457,10 +457,6 @@ object CarbonDataRDDFactory {
// Check if any load need to be deleted before loading new data
DataManagementFunc.deleteLoadsAndUpdateMetadata(carbonLoadModel.getDatabaseName,
carbonLoadModel.getTableName, storePath, false, carbonTable)
- if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
- }
-
var currentLoadCount = -1
val convLoadDetails = carbonLoadModel.getLoadMetadataDetails.asScala
// taking the latest segment ID present.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2aa5d83b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index e83d541..0064c21 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -495,8 +495,17 @@ case class LoadTable(
maxColumns)
carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
GlobalDictionaryUtil.updateTableMetadataFunc = LoadTable.updateTableMetadata
+ val storePath = relation.tableMeta.storePath
+ if (null == carbonLoadModel.getLoadMetadataDetails) {
+ CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
+ }
+ if (carbonLoadModel.getLoadMetadataDetails.isEmpty && carbonLoadModel.getUseOnePass &&
+ StringUtils.isEmpty(columnDict) && StringUtils.isEmpty(allDictionaryPath)) {
+ LOGGER.info(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
+ LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
+ carbonLoadModel.setUseOnePass(false)
+ }
if (carbonLoadModel.getUseOnePass) {
- val storePath = relation.tableMeta.storePath
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
.getCarbonTableIdentifier
@@ -551,7 +560,6 @@ case class LoadTable(
} else {
None
}
-
CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
carbonLoadModel,
relation.tableMeta.storePath,