You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2020/06/01 09:44:07 UTC
[carbondata] annotated tag apache-carbondata-2.0.1-rc1 created (now
8569d80)
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a change to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git.
at 8569d80 (tag)
tagging c2889fea484aea381ccc0e9e0c6c9cc14c22cfdc (commit)
replaces apache-carbondata-2.0.0-rc3
by kunal642
on Mon Jun 1 15:13:51 2020 +0530
- Log -----------------------------------------------------------------
[maven-release-plugin] copy for tag apache-carbondata-2.0.1-rc1
-----------------------------------------------------------------------
This annotated tag includes the following new commits:
new 613d458 [maven-release-plugin] prepare for next development iteration
new aefa10a [HOTFIX] changed development version to 2.0.1
new a3a27d2 [CARBONDATA-3835] Fix global sort issues
new 86a6e88 [CARBONDATA-3837] Fallback to the original plan when mv rewrite throw exception
new be50362 [CARBONDATA-3836] Fix metadata folder FileNotFoundException while creating new carbon table
new 316ea18 [CARBONDATA-3839]Fix rename file failed for FilterFileSystem DFS object
new 8947655 [CARBONDATA-3840] Mark features as experimental
new c2889fe [maven-release-plugin] prepare release apache-carbondata-2.0.1-rc1
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[carbondata] 04/08: [CARBONDATA-3837] Fallback to the original plan
when mv rewrite throw exception
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit 86a6e88916eeb5995e65787021d600fa38cd17da
Author: QiangCai <qi...@qq.com>
AuthorDate: Mon Jun 1 00:01:47 2020 +0800
[CARBONDATA-3837] Fallback to the original plan when mv rewrite throw exception
Why is this PR needed?
All plans are checking MVRewriteRule,
if MVRewriteRule throw an exception, it will lead to query failure
What changes were proposed in this PR?
Only the query should check MVRewriteRule, other plans should skip it quickly.
catch all exceptions of MVRewriteRule, and fallback to original plan.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3777
---
.../apache/carbondata/core/view/MVProvider.java | 20 ++++++++++++++---
.../apache/carbondata/view/MVCatalogInSpark.scala | 25 +++++++++++-----------
.../apache/spark/sql/optimizer/MVRewriteRule.scala | 21 +++++++++++++++---
3 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java b/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
index 429f274..1259f91 100644
--- a/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/view/MVProvider.java
@@ -149,14 +149,28 @@ public class MVProvider {
private String getStatusFileName(MVManager viewManager, String databaseName) {
String databaseLocation = viewManager.getDatabaseLocation(databaseName);
- return FileFactory.getCarbonFile(databaseLocation).getCanonicalPath() +
- CarbonCommonConstants.FILE_SEPARATOR + "_system" +
- CarbonCommonConstants.FILE_SEPARATOR + STATUS_FILE_NAME;
+ try {
+ if (FileFactory.isFileExist(databaseLocation)) {
+ return FileFactory.getCarbonFile(databaseLocation).getCanonicalPath()
+ + CarbonCommonConstants.FILE_SEPARATOR + "_system"
+ + CarbonCommonConstants.FILE_SEPARATOR + STATUS_FILE_NAME;
+ } else {
+ // this database folder is not exists
+ return null;
+ }
+ } catch (IOException e) {
+ // avoid to impact other query on all databases because of mv failure on this database
+ LOG.warn("Failed to get mv status file for database " + databaseName, e);
+ return null;
+ }
}
public List<MVStatusDetail> getStatusDetails(MVManager viewManager, String databaseName)
throws IOException {
String statusPath = this.getStatusFileName(viewManager, databaseName);
+ if (statusPath == null) {
+ return Collections.emptyList();
+ }
Gson gsonObjectToRead = new Gson();
DataInputStream dataInputStream = null;
BufferedReader buffReader = null;
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/view/MVCatalogInSpark.scala b/integration/spark/src/main/scala/org/apache/carbondata/view/MVCatalogInSpark.scala
index b75a88b..c56b7e1 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/view/MVCatalogInSpark.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/view/MVCatalogInSpark.scala
@@ -90,18 +90,19 @@ case class MVCatalogInSpark(session: SparkSession)
def registerSchema(mvSchema: MVSchema): Unit = {
withWriteLock {
val currentDatabase = session.catalog.currentDatabase
-
- // This is required because mv schemas are across databases, so while loading the
- // catalog, if the mv is in database other than sparkSession.currentDataBase(), then it
- // fails to register, so set the database present in the mvSchema Object
- session.catalog.setCurrentDatabase(mvSchema.getIdentifier.getDatabaseName)
- val logicalPlan = MVHelper.dropDummyFunction(
- MVQueryParser.getQueryPlan(mvSchema.getQuery, session))
- // here setting back to current database of current session, because if the actual query
- // contains db name in query like, select db1.column1 from table and current database is
- // default and if we drop the db1, still the session has current db as db1.
- // So setting back to current database.
- session.catalog.setCurrentDatabase(currentDatabase)
+ val logicalPlan = try {
+ // This is required because mv schemas are across databases, so while loading the
+ // catalog, if the mv is in database other than sparkSession.currentDataBase(), then it
+ // fails to register, so set the database present in the mvSchema Object
+ session.catalog.setCurrentDatabase(mvSchema.getIdentifier.getDatabaseName)
+ MVHelper.dropDummyFunction(MVQueryParser.getQueryPlan(mvSchema.getQuery, session))
+ } finally {
+ // here setting back to current database of current session, because if the actual query
+ // contains db name in query like, select db1.column1 from table and current database is
+ // default and if we drop the db1, still the session has current db as db1.
+ // So setting back to current database.
+ session.catalog.setCurrentDatabase(currentDatabase)
+ }
val mvSignature = SimpleModularizer.modularize(
BirdcageOptimizer.execute(logicalPlan)).next().semiHarmonized.signature
val mvIdentifier = mvSchema.getIdentifier
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewriteRule.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewriteRule.scala
index 4bcca7d..f475a2b 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewriteRule.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/MVRewriteRule.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedAttribute}
import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation}
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, ScalaUDF}
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Command, DeserializeToObject, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Command, DeserializeToObject, LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -39,8 +39,6 @@ import org.apache.carbondata.view.MVFunctions.DUMMY_FUNCTION
*/
class MVRewriteRule(session: SparkSession) extends Rule[LogicalPlan] {
- private val logger = MVRewriteRule.LOGGER
-
private val catalogFactory = new MVCatalogFactory[MVSchemaWrapper] {
override def newCatalog(): MVCatalog[MVSchemaWrapper] = {
new MVCatalogInSpark(session)
@@ -48,6 +46,23 @@ class MVRewriteRule(session: SparkSession) extends Rule[LogicalPlan] {
}
override def apply(logicalPlan: LogicalPlan): LogicalPlan = {
+ // only query need to check this rule
+ logicalPlan match {
+ case _: Command => return logicalPlan
+ case _: LocalRelation => return logicalPlan
+ case _ =>
+ }
+ try {
+ tryRewritePlan(logicalPlan)
+ } catch {
+ case e =>
+ // if exception is thrown while rewriting the query, will fallback to original query plan.
+ MVRewriteRule.LOGGER.warn("Failed to rewrite plan with mv: " + e.getMessage)
+ logicalPlan
+ }
+ }
+
+ private def tryRewritePlan(logicalPlan: LogicalPlan): LogicalPlan = {
var canApply = true
logicalPlan.transformAllExpressions {
// first check if any mv UDF is applied it is present is in plan
[carbondata] 02/08: [HOTFIX] changed development version to 2.0.1
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit aefa10aee89c78101262b27f191c5758bd8532b8
Author: kunal642 <ku...@gmail.com>
AuthorDate: Sat May 30 11:31:36 2020 +0530
[HOTFIX] changed development version to 2.0.1
---
assembly/pom.xml | 2 +-
common/pom.xml | 2 +-
core/pom.xml | 2 +-
examples/flink/pom.xml | 2 +-
examples/spark/pom.xml | 2 +-
format/pom.xml | 2 +-
geo/pom.xml | 2 +-
hadoop/pom.xml | 2 +-
index/bloom/pom.xml | 2 +-
index/examples/pom.xml | 2 +-
index/lucene/pom.xml | 2 +-
index/secondary-index/pom.xml | 2 +-
integration/flink-build/pom.xml | 2 +-
integration/flink-proxy/pom.xml | 2 +-
integration/flink/pom.xml | 2 +-
integration/hive/pom.xml | 2 +-
integration/presto/pom.xml | 2 +-
integration/spark-common-cluster-test/pom.xml | 2 +-
integration/spark/pom.xml | 2 +-
mv/plan/pom.xml | 2 +-
pom.xml | 2 +-
processing/pom.xml | 2 +-
sdk/sdk/pom.xml | 2 +-
streaming/pom.xml | 2 +-
tools/cli/pom.xml | 2 +-
25 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 425e3e7..25eac29 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/common/pom.xml b/common/pom.xml
index bbf1593..c376e6e 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/core/pom.xml b/core/pom.xml
index 5cf11a7..4eed3da 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 8f4f371..bb1fe3a 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 715732c..8753d92 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/format/pom.xml b/format/pom.xml
index 9017f4b..a0ca71d 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/geo/pom.xml b/geo/pom.xml
index 606089f..d184910 100644
--- a/geo/pom.xml
+++ b/geo/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 69cc55a..0b7a57a 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml
index e58ac25..1948c47 100644
--- a/index/bloom/pom.xml
+++ b/index/bloom/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/examples/pom.xml b/index/examples/pom.xml
index aa2030f..b6f338b 100644
--- a/index/examples/pom.xml
+++ b/index/examples/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml
index 3dff9e7..2d14e7b 100644
--- a/index/lucene/pom.xml
+++ b/index/lucene/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml
index c363c65..c05b334 100644
--- a/index/secondary-index/pom.xml
+++ b/index/secondary-index/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml
index 70aa992..647a523 100644
--- a/integration/flink-build/pom.xml
+++ b/integration/flink-build/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml
index dc91066..3d375f7 100644
--- a/integration/flink-proxy/pom.xml
+++ b/integration/flink-proxy/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml
index 191aea0..4b68420 100644
--- a/integration/flink/pom.xml
+++ b/integration/flink/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index a2667f5..6baa4a7 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index f677742..4870ae0 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index 12426ff..ba550ec 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
index b5f2b9d..ee3f625 100644
--- a/integration/spark/pom.xml
+++ b/integration/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml
index 11e7120..e7bcb79 100644
--- a/mv/plan/pom.xml
+++ b/mv/plan/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index 49ad5b5..5489aea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
<inceptionYear>2016</inceptionYear>
<packaging>pom</packaging>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<licenses>
<license>
diff --git a/processing/pom.xml b/processing/pom.xml
index 49d7640..0706767 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdk/sdk/pom.xml b/sdk/sdk/pom.xml
index fcd9062..eeeb8be 100644
--- a/sdk/sdk/pom.xml
+++ b/sdk/sdk/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 92cbed4..a546ad7 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -2,7 +2,7 @@
<parent>
<artifactId>carbondata-parent</artifactId>
<groupId>org.apache.carbondata</groupId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml
index f851e4c..f5c3654 100644
--- a/tools/cli/pom.xml
+++ b/tools/cli/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
[carbondata] 08/08: [maven-release-plugin] prepare release
apache-carbondata-2.0.1-rc1
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit c2889fea484aea381ccc0e9e0c6c9cc14c22cfdc
Author: kunal642 <ku...@gmail.com>
AuthorDate: Mon Jun 1 15:12:49 2020 +0530
[maven-release-plugin] prepare release apache-carbondata-2.0.1-rc1
---
assembly/pom.xml | 2 +-
common/pom.xml | 2 +-
core/pom.xml | 2 +-
examples/flink/pom.xml | 2 +-
examples/spark/pom.xml | 2 +-
format/pom.xml | 2 +-
geo/pom.xml | 2 +-
hadoop/pom.xml | 2 +-
index/bloom/pom.xml | 2 +-
index/examples/pom.xml | 2 +-
index/lucene/pom.xml | 2 +-
index/secondary-index/pom.xml | 2 +-
integration/flink-build/pom.xml | 2 +-
integration/flink-proxy/pom.xml | 2 +-
integration/flink/pom.xml | 2 +-
integration/hive/pom.xml | 2 +-
integration/presto/pom.xml | 2 +-
integration/spark/pom.xml | 2 +-
mv/plan/pom.xml | 2 +-
pom.xml | 4 ++--
processing/pom.xml | 2 +-
sdk/sdk/pom.xml | 2 +-
streaming/pom.xml | 2 +-
tools/cli/pom.xml | 2 +-
24 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 25eac29..bae9c4c 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/common/pom.xml b/common/pom.xml
index c376e6e..e6f86c3 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/core/pom.xml b/core/pom.xml
index 4eed3da..eb237f9 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index bb1fe3a..cd7a3d6 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index 8753d92..6bda3d6 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/format/pom.xml b/format/pom.xml
index a0ca71d..61cd65f 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/geo/pom.xml b/geo/pom.xml
index d184910..e1a0ff1 100644
--- a/geo/pom.xml
+++ b/geo/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 0b7a57a..b178e7a 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml
index 1948c47..28210d0 100644
--- a/index/bloom/pom.xml
+++ b/index/bloom/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/examples/pom.xml b/index/examples/pom.xml
index b6f338b..10978ac 100644
--- a/index/examples/pom.xml
+++ b/index/examples/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml
index 2d14e7b..1aa242f 100644
--- a/index/lucene/pom.xml
+++ b/index/lucene/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml
index c05b334..a024262 100644
--- a/index/secondary-index/pom.xml
+++ b/index/secondary-index/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml
index 647a523..cf867ac 100644
--- a/integration/flink-build/pom.xml
+++ b/integration/flink-build/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml
index 3d375f7..1a6e7fa 100644
--- a/integration/flink-proxy/pom.xml
+++ b/integration/flink-proxy/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml
index 4b68420..b9837b1 100644
--- a/integration/flink/pom.xml
+++ b/integration/flink/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 6baa4a7..f7e4d59 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 4870ae0..7450101 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
index ee3f625..fa62578 100644
--- a/integration/spark/pom.xml
+++ b/integration/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml
index e7bcb79..6a83b39 100644
--- a/mv/plan/pom.xml
+++ b/mv/plan/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index 5489aea..2f8af67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
<inceptionYear>2016</inceptionYear>
<packaging>pom</packaging>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<licenses>
<license>
@@ -49,7 +49,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/carbondata.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/carbondata.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/carbondata.git</url>
- <tag>HEAD</tag>
+ <tag>apache-carbondata-2.0.1-rc1</tag>
</scm>
<issueManagement>
diff --git a/processing/pom.xml b/processing/pom.xml
index 0706767..65585e5 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdk/sdk/pom.xml b/sdk/sdk/pom.xml
index eeeb8be..90e3bd7 100644
--- a/sdk/sdk/pom.xml
+++ b/sdk/sdk/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index a546ad7..43dca5f 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -2,7 +2,7 @@
<parent>
<artifactId>carbondata-parent</artifactId>
<groupId>org.apache.carbondata</groupId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml
index f5c3654..1d20208 100644
--- a/tools/cli/pom.xml
+++ b/tools/cli/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.1-SNAPSHOT</version>
+ <version>2.0.1</version>
<relativePath>../../pom.xml</relativePath>
</parent>
[carbondata] 05/08: [CARBONDATA-3836] Fix metadata folder
FileNotFoundException while creating new carbon table
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit be50362f14511e014a45b2eff7ecffd579a3b877
Author: Manhua <ke...@qq.com>
AuthorDate: Sat May 30 10:47:10 2020 +0800
[CARBONDATA-3836] Fix metadata folder FileNotFoundException while creating new carbon table
Why is this PR needed?
1. In the case of using carbon only setting carbon.storelocation, carbon will use local spark warehouse path instead of the configured value.
2. FileNotFoundException is thrown when creating schema file for a brand new table. Because current implementation gets the schema file path by listing the Metadata directory which has not been created.
What changes were proposed in this PR?
1. spark.sql.warehouse.dir has its own default value in Spark, remove using carbonStorePath as default value, which will make hiveStorePath.equals(carbonStorePath) TRUE when user not set spark.sql.warehouse.dir.
2. create the Metadata directory before getting the schema file path.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3780
---
integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala | 2 +-
.../main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 9df5809..5062a43 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -328,7 +328,7 @@ object CarbonEnv {
if ((!EnvHelper.isLegacy(sparkSession)) &&
(dbName.equals("default") || databaseLocation.endsWith(".db"))) {
val carbonStorePath = CarbonProperties.getStorePath()
- val hiveStorePath = sparkSession.conf.get("spark.sql.warehouse.dir", carbonStorePath)
+ val hiveStorePath = sparkSession.conf.get("spark.sql.warehouse.dir")
// if carbon.store does not point to spark.sql.warehouse.dir then follow the old table path
// format
if (carbonStorePath != null && !hiveStorePath.equals(carbonStorePath)) {
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 5156798..4c5f16d 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -402,8 +402,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
private def createSchemaThriftFile(
identifier: AbsoluteTableIdentifier,
thriftTableInfo: TableInfo): String = {
- val schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath)
- val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
+ val schemaMetadataPath = CarbonTablePath.getMetadataPath(identifier.getTablePath)
if (!FileFactory.isFileExist(schemaMetadataPath)) {
val isDirCreated = FileFactory
.mkdirs(schemaMetadataPath, SparkSession.getActiveSession.get.sessionState.newHadoopConf())
@@ -411,6 +410,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
throw new IOException(s"Failed to create the metadata directory $schemaMetadataPath")
}
}
+ val schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath)
val thriftWriter = new ThriftWriter(schemaFilePath, false)
thriftWriter.open(FileWriteOperation.OVERWRITE)
thriftWriter.write(thriftTableInfo)
[carbondata] 01/08: [maven-release-plugin] prepare for next
development iteration
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit 613d458e85fe392d7bc2ce579285cae1be2e42a7
Author: kunal642 <ku...@gmail.com>
AuthorDate: Sun May 17 13:28:30 2020 +0530
[maven-release-plugin] prepare for next development iteration
---
assembly/pom.xml | 2 +-
common/pom.xml | 2 +-
core/pom.xml | 2 +-
examples/flink/pom.xml | 2 +-
examples/spark/pom.xml | 2 +-
format/pom.xml | 2 +-
geo/pom.xml | 2 +-
hadoop/pom.xml | 2 +-
index/bloom/pom.xml | 2 +-
index/examples/pom.xml | 2 +-
index/lucene/pom.xml | 2 +-
index/secondary-index/pom.xml | 2 +-
integration/flink-build/pom.xml | 2 +-
integration/flink-proxy/pom.xml | 2 +-
integration/flink/pom.xml | 2 +-
integration/hive/pom.xml | 2 +-
integration/presto/pom.xml | 2 +-
integration/spark/pom.xml | 2 +-
mv/plan/pom.xml | 2 +-
pom.xml | 4 ++--
processing/pom.xml | 2 +-
sdk/sdk/pom.xml | 2 +-
streaming/pom.xml | 2 +-
tools/cli/pom.xml | 2 +-
24 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index ca3f10c..425e3e7 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/common/pom.xml b/common/pom.xml
index f055fc3..bbf1593 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/core/pom.xml b/core/pom.xml
index 8414b00..5cf11a7 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 57e8398..8f4f371 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index b96afbd..715732c 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/format/pom.xml b/format/pom.xml
index 9aaa631..9017f4b 100644
--- a/format/pom.xml
+++ b/format/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/geo/pom.xml b/geo/pom.xml
index f614997..606089f 100644
--- a/geo/pom.xml
+++ b/geo/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 24be836..69cc55a 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/index/bloom/pom.xml b/index/bloom/pom.xml
index 78fd642..e58ac25 100644
--- a/index/bloom/pom.xml
+++ b/index/bloom/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/examples/pom.xml b/index/examples/pom.xml
index f32e010..aa2030f 100644
--- a/index/examples/pom.xml
+++ b/index/examples/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/lucene/pom.xml b/index/lucene/pom.xml
index 4d00226..3dff9e7 100644
--- a/index/lucene/pom.xml
+++ b/index/lucene/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/index/secondary-index/pom.xml b/index/secondary-index/pom.xml
index 626985f..c363c65 100644
--- a/index/secondary-index/pom.xml
+++ b/index/secondary-index/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-build/pom.xml b/integration/flink-build/pom.xml
index 26f0872..70aa992 100644
--- a/integration/flink-build/pom.xml
+++ b/integration/flink-build/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink-proxy/pom.xml b/integration/flink-proxy/pom.xml
index 22e305d..dc91066 100644
--- a/integration/flink-proxy/pom.xml
+++ b/integration/flink-proxy/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/flink/pom.xml b/integration/flink/pom.xml
index 4ce5617..191aea0 100644
--- a/integration/flink/pom.xml
+++ b/integration/flink/pom.xml
@@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/hive/pom.xml b/integration/hive/pom.xml
index 9421594..a2667f5 100644
--- a/integration/hive/pom.xml
+++ b/integration/hive/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 8b7cd0e..f677742 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml
index 9c3aa7d..b5f2b9d 100644
--- a/integration/spark/pom.xml
+++ b/integration/spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/mv/plan/pom.xml b/mv/plan/pom.xml
index 9144c3e..11e7120 100644
--- a/mv/plan/pom.xml
+++ b/mv/plan/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index 66f2146..49ad5b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
<inceptionYear>2016</inceptionYear>
<packaging>pom</packaging>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<licenses>
<license>
@@ -49,7 +49,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/carbondata.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/carbondata.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/carbondata.git</url>
- <tag>apache-carbondata-2.0.0-rc3</tag>
+ <tag>HEAD</tag>
</scm>
<issueManagement>
diff --git a/processing/pom.xml b/processing/pom.xml
index 9eb53f3..49d7640 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sdk/sdk/pom.xml b/sdk/sdk/pom.xml
index 147b49c..fcd9062 100644
--- a/sdk/sdk/pom.xml
+++ b/sdk/sdk/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 3b627c7..92cbed4 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -2,7 +2,7 @@
<parent>
<artifactId>carbondata-parent</artifactId>
<groupId>org.apache.carbondata</groupId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml
index d842523..f851e4c 100644
--- a/tools/cli/pom.xml
+++ b/tools/cli/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>2.0.0</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
[carbondata] 07/08: [CARBONDATA-3840] Mark features as experimental
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit 8947655194db7c76987e97ebb5d01bbadb587b8a
Author: kunal642 <ku...@gmail.com>
AuthorDate: Mon Jun 1 13:07:24 2020 +0530
[CARBONDATA-3840] Mark features as experimental
Why is this PR needed?
Mark features as experimental because they are subject to change in future.
What changes were proposed in this PR?
Mark features as experimental because they are subject to change in future.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3783
---
README.md | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/README.md b/README.md
index 2d9fcc7..0293c4c 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,14 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com
* [Carbon as Spark's Datasource](https://github.com/apache/carbondata/blob/master/docs/carbon-as-spark-datasource-guide.md)
* [FAQs](https://github.com/apache/carbondata/blob/master/docs/faq.md)
+## Experimental Features
+
+Some features are marked as experimental because the syntax/implementation might change in the future.
+1. Hybrid format table using Add Segment.
+2. Accelerating performance using MV on parquet/orc.
+3. Merge API for Spark DataFrame.
+4. Hive write for non-transactional table.
+
## Integration
* [Hive](https://github.com/apache/carbondata/blob/master/docs/hive-guide.md)
* [Presto](https://github.com/apache/carbondata/blob/master/docs/prestodb-guide.md)
[carbondata] 06/08: [CARBONDATA-3839]Fix rename file failed for
FilterFileSystem DFS object
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit 316ea18a8e3825090f1e9083b67dee278bb9a98c
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Mon Jun 1 09:58:31 2020 +0530
[CARBONDATA-3839]Fix rename file failed for FilterFileSystem DFS object
Why is this PR needed?
Rename file fails in HDFS when the FS object is of FilterFileSystem,
(which basically can contain any filesystem to use as basic filesystem)
What changes were proposed in this PR?
While rename force, have a check for this FS object
This closes #3781
---
.../apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index ac508c3..7d36f0e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -23,6 +23,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.log4j.Logger;
@@ -77,6 +78,11 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
((DistributedFileSystem) fileSystem).rename(path, new Path(changetoName),
org.apache.hadoop.fs.Options.Rename.OVERWRITE);
return true;
+ } else if ((fileSystem instanceof FilterFileSystem) && (((FilterFileSystem) fileSystem)
+ .getRawFileSystem() instanceof DistributedFileSystem)) {
+ ((DistributedFileSystem) ((FilterFileSystem) fileSystem).getRawFileSystem())
+ .rename(path, new Path(changetoName), org.apache.hadoop.fs.Options.Rename.OVERWRITE);
+ return true;
} else {
return fileSystem.rename(path, new Path(changetoName));
}
[carbondata] 03/08: [CARBONDATA-3835] Fix global sort issues
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to annotated tag apache-carbondata-2.0.1-rc1
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit a3a27d219d020ae3b0ba6a4565b88758b613c6c4
Author: ajantha-bhat <aj...@gmail.com>
AuthorDate: Sat May 30 08:24:39 2020 +0530
[CARBONDATA-3835] Fix global sort issues
Why is this PR needed?
For global sort without partition, string comes as byte[], if we use string comparator (StringSerializableComparator) it will convert byte[] to toString which gives address and comparison goes wrong.
For global sort with partition, when sort column is partition column, it was sorting on first column instead of partition columns.
What changes were proposed in this PR?
change data type to byte before choosing a comparator.
get the sorted column based on index, don't just take from first
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3779
---
.../spark/load/DataLoadProcessBuilderOnSpark.scala | 15 ++++--
.../carbondata/spark/load/GlobalSortHelper.scala | 8 ++-
.../command/management/CommonLoadUtils.scala | 7 ++-
.../org/apache/spark/sql/test/util/QueryTest.scala | 17 +++++-
.../StandardPartitionGlobalSortTestCase.scala | 60 ++++++++++++++++++++++
5 files changed, 94 insertions(+), 13 deletions(-)
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index 55eee11..e7e1baf 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.command.ExecutionErrors
+import org.apache.spark.sql.types.{ByteType, DateType, LongType, StringType, TimestampType}
import org.apache.spark.sql.util.{SparkSQLUtil, SparkTypeConverter}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.unsafe.types.UTF8String
@@ -237,10 +238,15 @@ object DataLoadProcessBuilderOnSpark {
CarbonProperties.getInstance().getGlobalSortRddStorageLevel()))
}
val sortColumnsLength = model.getCarbonDataLoadSchema.getCarbonTable.getSortColumns.size()
- val sortColumnDataTypes = dataTypes.take(sortColumnsLength)
- val rowComparator = GlobalSortHelper.generateRowComparator(sortColumnDataTypes)
+ var sortColumnDataTypes = dataTypes.take(sortColumnsLength)
+ sortColumnDataTypes = sortColumnDataTypes.map {
+ case StringType => ByteType
+ case TimestampType | DateType => LongType
+ case datatype => datatype
+ }
+ val rowComparator = GlobalSortHelper.generateRowComparator(sortColumnDataTypes.zipWithIndex)
val sortRDD = rdd.sortBy(row =>
- getKey(row, sortColumnsLength, sortColumnDataTypes),
+ getKey(row, sortColumnsLength),
true,
numPartitions)(
rowComparator, classTag[Array[AnyRef]])
@@ -273,8 +279,7 @@ object DataLoadProcessBuilderOnSpark {
}
def getKey(row: Array[AnyRef],
- sortColumnsLength: Int,
- dataTypes: Seq[org.apache.spark.sql.types.DataType]): Array[AnyRef] = {
+ sortColumnsLength: Int): Array[AnyRef] = {
val key: Array[AnyRef] = new Array[AnyRef](sortColumnsLength)
System.arraycopy(row, 0, key, 0, sortColumnsLength)
key
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/GlobalSortHelper.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/GlobalSortHelper.scala
index 91ed27e..00891b9 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/load/GlobalSortHelper.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/load/GlobalSortHelper.scala
@@ -50,7 +50,7 @@ object GlobalSortHelper {
def sortBy(updatedRdd: RDD[InternalRow],
numPartitions: Int,
- dataTypes: Seq[DataType]
+ dataTypes: Seq[(DataType, Int)]
): RDD[InternalRow] = {
val keyExtractors = generateKeyExtractor(dataTypes)
val rowComparator = generateRowComparator(dataTypes)
@@ -68,9 +68,8 @@ object GlobalSortHelper {
key
}
- def generateKeyExtractor(dataTypes: Seq[DataType]): Array[KeyExtractor] = {
+ def generateKeyExtractor(dataTypes: Seq[(DataType, Int)]): Array[KeyExtractor] = {
dataTypes
- .zipWithIndex
.map { attr =>
attr._1 match {
case StringType => UTF8StringKeyExtractor(attr._2)
@@ -91,9 +90,8 @@ object GlobalSortHelper {
.toArray
}
- def generateRowComparator(dataTypes: Seq[DataType]): InternalRowComparator = {
+ def generateRowComparator(dataTypes: Seq[(DataType, Int)]): InternalRowComparator = {
val comparators = dataTypes
- .zipWithIndex
.map { attr =>
val comparator = attr._1 match {
case StringType => new StringSerializableComparator()
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
index ce699d6..20d29d8 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala
@@ -518,7 +518,12 @@ object CommonLoadUtils {
} else {
attributes.take(table.getSortColumns.size())
}
- val dataTypes = sortColumns.map(_.dataType)
+ val attributesWithIndex = attributes.zipWithIndex
+ val dataTypes = sortColumns.map { column =>
+ val attributeWithIndex =
+ attributesWithIndex.find(x => x._1.name.equalsIgnoreCase(column.name))
+ (column.dataType, attributeWithIndex.get._2)
+ }
val sortedRDD: RDD[InternalRow] =
GlobalSortHelper.sortBy(updatedRdd, numPartitions, dataTypes)
val outputOrdering = sortColumns.map(SortOrder(_, Ascending))
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
index ef9d1b2..c08370b 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala
@@ -90,6 +90,13 @@ class QueryTest extends PlanTest {
}
}
+ protected def checkAnswerWithoutSort(df: DataFrame, expectedAnswer: Seq[Row]): Unit = {
+ QueryTest.checkAnswer(df, expectedAnswer, needSort = false) match {
+ case Some(errorMessage) => fail(errorMessage)
+ case None =>
+ }
+ }
+
protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = {
checkAnswer(df, Seq(expectedAnswer))
}
@@ -219,8 +226,14 @@ object QueryTest {
* @param df the [[DataFrame]] to be executed
* @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s.
*/
- def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = {
- val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty
+ def checkAnswer(df: DataFrame,
+ expectedAnswer: Seq[Row],
+ needSort: Boolean = true): Option[String] = {
+ val isSorted = if (needSort) {
+ df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty
+ } else {
+ true
+ }
def prepareAnswer(answer: Seq[Row]): Seq[Row] = {
// Converts data to types that we can do equality comparison using Scala collections.
// For BigDecimal type, the Scala type has a better definition of equality test (similar to
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
index 3deb8ba..a9c773d 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
@@ -48,6 +48,66 @@ class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterA
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE originTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""")
}
+ test("test global sort column as partition column") {
+ sql("drop table if exists table1")
+ sql(s"""
+ | create table table1(
+ | timestampField TIMESTAMP
+ | )
+ | stored as carbondata
+ | tblproperties('sort_scope'='global_sort', 'sort_columns'='charField')
+ | partitioned by (charField String)
+ """.stripMargin)
+ sql(
+ s"""
+ | INSERT INTO TABLE table1
+ | SELECT
+ | '2015-07-01 00:00:00', 'aaa'
+ """.stripMargin)
+ checkAnswer(sql(s"SELECT charField FROM table1"), Seq(Row("aaa")))
+ sql("drop table if exists table1")
+ }
+
+ test("test global sort order") {
+ sql("DROP TABLE IF EXISTS gs_test")
+ sql("create table gs_test (id string) " +
+ "using carbondata options('sort_scope'='global_sort', 'local_dictionary_enable'='false', 'sort_columns'='id','global_sort_partitions'='1')")
+ sql("insert into gs_test select 'abc1' union all select 'abc5' union all select 'abc10' union all select 'abc20' ")
+ checkAnswerWithoutSort(sql("select * from gs_test"), Seq(Row("abc1"), Row("abc10"), Row("abc20"), Row("abc5")))
+ sql("DROP TABLE IF EXISTS gs_test")
+ }
+
+ test("test global sort order in old insert flow") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "true");
+ sql("DROP TABLE IF EXISTS gs_test")
+ sql("create table gs_test (id string) " +
+ "using carbondata options('sort_scope'='global_sort', 'local_dictionary_enable'='false', 'sort_columns'='id','global_sort_partitions'='1')")
+ sql("insert into gs_test select 'abc1' union all select 'abc5' union all select 'abc10' union all select 'abc20' ")
+ checkAnswerWithoutSort(sql("select id from gs_test"), Seq(Row("abc1"), Row("abc10"), Row("abc20"), Row("abc5")))
+ sql("DROP TABLE IF EXISTS gs_test")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "false");
+ }
+
+ test("test global sort partition order") {
+ sql("DROP TABLE IF EXISTS gs_test")
+ sql("create table gs_test (id string) partitioned by (val int)" +
+ "stored as carbondata tblproperties('sort_scope'='global_sort', 'local_dictionary_enable'='false', 'sort_columns'='id','global_sort_partitions'='1')")
+ sql("insert into gs_test select 'abc1', 1 union all select 'abc5', 1 union all select 'abc10', 1 union all select 'abc20', 1 ")
+ checkAnswerWithoutSort(sql("select id from gs_test"), Seq(Row("abc1"), Row("abc10"), Row("abc20"), Row("abc5")))
+ sql("DROP TABLE IF EXISTS gs_test")
+ }
+
+ test("test global sort order in old insert partition flow") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "true");
+ sql("DROP TABLE IF EXISTS gs_test")
+ sql("create table gs_test (id string) " +
+ "using carbondata options('sort_scope'='global_sort', 'local_dictionary_enable'='false', 'sort_columns'='id','global_sort_partitions'='1')")
+ sql("insert into gs_test select 'abc1' union all select 'abc5' union all select 'abc10' union all select 'abc20' ")
+ checkAnswerWithoutSort(sql("select id from gs_test"), Seq(Row("abc1"), Row("abc10"), Row("abc20"), Row("abc5")))
+ sql("DROP TABLE IF EXISTS gs_test")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "false");
+ }
+
test("data loading for global sort partition table for one partition column") {
sql(
"""