You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/08/07 10:21:48 UTC
carbondata git commit: [CARBONDATA-2823] Support streaming property
with datamap
Repository: carbondata
Updated Branches:
refs/heads/master abcd4f6e2 -> b9e510640
[CARBONDATA-2823] Support streaming property with datamap
Since during query, carbondata get splits from streaming segment and
columnar segments repectively, we can support streaming with index
datamap.
For preaggregate datamap, it already supported streaming table, so here
we will remove the outdated comments.
This closes #2609
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b9e51064
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b9e51064
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b9e51064
Branch: refs/heads/master
Commit: b9e51064017e428962d86ff8fccb3c1493e9864b
Parents: abcd4f6
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Aug 6 15:34:51 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Aug 7 18:21:30 2018 +0800
----------------------------------------------------------------------
.../datamap/bloom/BloomCoarseGrainDataMapFactory.java | 2 +-
.../datamap/lucene/LuceneFineGrainDataMapFactory.java | 2 +-
.../apache/spark/sql/execution/strategy/DDLStrategy.scala | 8 --------
3 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 80a86cc..0d240c4 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -391,7 +391,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
case ALTER_CHANGE_DATATYPE:
return true;
case STREAMING:
- return true;
+ return false;
case DELETE:
return true;
case UPDATE:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
index 8c7539f..ff2ffdd 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
@@ -98,7 +98,7 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine
case ALTER_CHANGE_DATATYPE:
return true;
case STREAMING:
- return true;
+ return false;
case DELETE:
return true;
case UPDATE:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b9e51064/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index 416dbec..94b988c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -35,7 +35,6 @@ import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.features.TableOperation
import org.apache.carbondata.core.util.CarbonProperties
/**
@@ -257,8 +256,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
if CarbonEnv.getInstance(sparkSession).carbonMetastore
.tableExists(tableName)(sparkSession) => {
- // TODO remove this limiation after streaming table support 'preaggregate' DataMap
- // if the table has 'preaggregate' DataMap, it doesn't support streaming now
val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
.lookupRelation(tableName)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
if (carbonTable != null && !carbonTable.getTableInfo.isTransactionalTable) {
@@ -266,11 +263,6 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
"Unsupported operation on non transactional table")
}
- if (carbonTable != null && !carbonTable.canAllow(carbonTable, TableOperation.STREAMING)) {
- throw new MalformedCarbonCommandException(
- "streaming is not supported for index datamap")
- }
-
// TODO remove this limitation later
val property = properties.find(_._1.equalsIgnoreCase("streaming"))
if (property.isDefined) {