You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/09 18:26:17 UTC
[33/47] carbondata git commit: [CARBONDATA-2823] Support streaming
property with datamap
[CARBONDATA-2823] Support streaming property with datamap
Since during query, carbondata get splits from streaming segment and
columnar segments repectively, we can support streaming with index
datamap.
For preaggregate datamap, it already supported streaming table, so here
we will remove the outdated comments.
This closes #2609
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/383f10e6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/383f10e6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/383f10e6
Branch: refs/heads/branch-1.4
Commit: 383f10e653405fd553c9891a3671920f6d6f3ba2
Parents: 5e547be
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Aug 6 15:34:51 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 9 23:51:17 2018 +0530
----------------------------------------------------------------------
.../datamap/bloom/BloomCoarseGrainDataMapFactory.java | 2 +-
.../datamap/lucene/LuceneFineGrainDataMapFactory.java | 2 +-
.../apache/spark/sql/execution/strategy/DDLStrategy.scala | 8 --------
3 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/383f10e6/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 80a86cc..0d240c4 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -391,7 +391,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
case ALTER_CHANGE_DATATYPE:
return true;
case STREAMING:
- return true;
+ return false;
case DELETE:
return true;
case UPDATE:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/383f10e6/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
index 8c7539f..ff2ffdd 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
@@ -98,7 +98,7 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine
case ALTER_CHANGE_DATATYPE:
return true;
case STREAMING:
- return true;
+ return false;
case DELETE:
return true;
case UPDATE:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/383f10e6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 416dbec..94b988c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -35,7 +35,6 @@ import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.features.TableOperation
import org.apache.carbondata.core.util.CarbonProperties
/**
@@ -257,8 +256,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
if CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(tableName)(sparkSession) => {
- // TODO remove this limiation after streaming table support 'preaggregate' DataMap
- // if the table has 'preaggregate' DataMap, it doesn't support streaming now
val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(tableName)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable) {
@@ -266,11 +263,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
"Unsupported operation on non transactional table")
}
- if (carbonTable != null && !carbonTable.canAllow(carbonTable, TableOperation.STREAMING)) {
- throw new MalformedCarbonCommandException(
- "streaming is not supported for index datamap")
- }
-
// TODO remove this limitation later
val property = properties.find(_._1.equalsIgnoreCase("streaming"))
if (property.isDefined) {