You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/08/07 12:48:02 UTC
carbondata git commit: [CARBONDATA-2768][CarbonStore] Fix error in
tests for external csv format
Repository: carbondata
Updated Branches:
refs/heads/external-format 0d2769f75 -> ccf64ce5a
[CARBONDATA-2768][CarbonStore] Fix error in tests for external csv format
In previous implementation earlier than PR2495, we only supportted csv as
external format for carbondata. And we validated the restriction while
creating the table.PR2495 added kafka support, so it removed the
validation, but it did not fix the related test case which cause failure in current version.
This PR fix the error test case.
This closes #2537
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ccf64ce5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ccf64ce5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ccf64ce5
Branch: refs/heads/external-format
Commit: ccf64ce5a9d17c053ee59e43cf0f580104ef9cd9
Parents: 0d2769f
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Fri Aug 3 14:34:32 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Aug 7 20:47:30 2018 +0800
----------------------------------------------------------------------
.../core/statusmanager/FileFormatProperties.java | 17 +++++++++++++++++
.../externalformat/CsvBasedCarbonTableSuite.scala | 2 +-
.../command/carbonTableSchemaCommon.scala | 16 +++++++++-------
3 files changed, 27 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ccf64ce5/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormatProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormatProperties.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormatProperties.java
index 862c36c..4372b44 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormatProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/FileFormatProperties.java
@@ -17,10 +17,27 @@
package org.apache.carbondata.core.statusmanager;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Provides the constant name for the file format properties
*/
public class FileFormatProperties {
+ private static final Set<String> SUPPORTED_EXTERNAL_FORMAT = new HashSet<String>() {
+ {
+ add("csv");
+ add("kafka");
+ }
+ };
+
+ public static boolean isExternalFormatSupported(String format) {
+ return SUPPORTED_EXTERNAL_FORMAT.contains(format.toLowerCase());
+ }
+ public static Set<String> getSupportedExternalFormat() {
+ return SUPPORTED_EXTERNAL_FORMAT;
+ }
+
public static class CSV {
public static final String HEADER = "csv.header";
public static final String DELIMITER = "csv.delimiter";
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ccf64ce5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/externalformat/CsvBasedCarbonTableSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/externalformat/CsvBasedCarbonTableSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/externalformat/CsvBasedCarbonTableSuite.scala
index 7f07878..85ccc10 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/externalformat/CsvBasedCarbonTableSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/externalformat/CsvBasedCarbonTableSuite.scala
@@ -151,7 +151,7 @@ class CsvBasedCarbonTableSuite extends QueryTest
)
}
- assert(expectedException.getMessage.contains("Currently we only support csv as external file format"))
+ assert(expectedException.getMessage.contains("Unsupported external format parquet"))
}
test("test csv based carbon table: the sequence of header does not match schema") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ccf64ce5/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 4f251e1..2d6a4be 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
+import org.apache.commons.lang3.StringUtils
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -39,12 +40,10 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema._
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier,
- TableInfo, TableSchema}
-import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema,
- ParentColumnTableRelation}
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier, TableInfo, TableSchema}
+import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema, ParentColumnTableRelation}
import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentUpdateStatusManager}
+import org.apache.carbondata.core.statusmanager.{FileFormatProperties, LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.loading.FailureCauses
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
@@ -859,8 +858,11 @@ class TableNewProcessor(cm: TableModel) {
tableInfo.setFactTable(tableSchema)
val format = cm.tableProperties.get(CarbonCommonConstants.FORMAT)
if (format.isDefined) {
- if (!format.get.equalsIgnoreCase("csv")) {
- CarbonException.analysisException(s"Currently we only support csv as external file format")
+ if (!FileFormatProperties.isExternalFormatSupported(format.get)) {
+ CarbonException.analysisException(
+ s"Unsupported external format ${format.get}, currently carbondata only support" +
+ s" ${FileFormatProperties.getSupportedExternalFormat.asScala.mkString(", ")}" +
+ s" as external file format")
}
tableInfo.setFormat(format.get)
val formatProperties = cm.tableProperties.filter(pair =>