You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/21 08:45:47 UTC
[1/2] incubator-carbondata git commit: fix load bug when table name
has '_'
Repository: incubator-carbondata
Updated Branches:
refs/heads/master a561d869c -> ddeb00425
fix load bug when table name has '_'
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fc1d620e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fc1d620e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fc1d620e
Branch: refs/heads/master
Commit: fc1d620e729746271c70159636e0cc05ac46d99e
Parents: a561d86
Author: Jay357089 <li...@huawei.com>
Authored: Wed Nov 16 10:49:35 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Wed Dec 21 16:43:20 2016 +0800
----------------------------------------------------------------------
.../core/carbon/path/CarbonTablePath.java | 8 ++--
.../core/constants/CarbonCommonConstants.java | 4 ++
.../dataload/TestLoadDataGeneral.scala | 42 +++++++++++++++++++-
3 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
index 99531e4..f90073e 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -397,10 +397,12 @@ public class CarbonTablePath extends Path {
*/
public static String getSegmentId(String dataFileAbsolutePath) {
// find segment id from last of data file path
- int endIndex = dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+ String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
+ CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
+ int endIndex = tempdataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
// + 1 for size of "/"
- int startIndex =
- dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
+ int startIndex = tempdataFileAbsolutePath.lastIndexOf(
+ CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
//identify id in segment_<id>
String[] segmentDirSplits = segmentDirStr.split("_");
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c32956c..2d1ba9f 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -461,6 +461,10 @@ public final class CarbonCommonConstants {
*/
public static final String POINT = ".";
/**
+ * Windows File separator
+ */
+ public static final String WINDOWS_FILE_SEPARATOR = "\\";
+ /**
* File separator
*/
public static final String FILE_SEPARATOR = "/";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 9904c93..6a274b6 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -22,11 +22,16 @@ package org.apache.carbondata.integration.spark.testsuite.dataload
import java.io.File
import java.math.BigDecimal
+import org.apache.carbondata.core.carbon.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.common.util.{CarbonHiveContext, QueryTest}
import org.scalatest.BeforeAndAfterAll
+import scala.collection.mutable.ArrayBuffer
+
class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
var currentDirectory: String = _
@@ -43,6 +48,24 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
.getCanonicalPath
}
+ private def checkSegmentExists(
+ segmentId: String,
+ datbaseName: String,
+ tableName: String): Boolean = {
+ val carbonTable = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
+ .getCarbonTable(datbaseName + "_" + tableName)
+ val partitionPath = CarbonStorePath.getCarbonTablePath(
+ CarbonHiveContext.hdfsCarbonBasePath,
+ carbonTable.getCarbonTableIdentifier).getPartitionDir("0")
+ val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
+ val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
+ val segments: ArrayBuffer[String] = ArrayBuffer()
+ carbonFile.listFiles.foreach { file =>
+ segments += CarbonTablePath.DataPathUtil.getSegmentId(file.getAbsolutePath + "/dummy")
+ }
+ segments.contains(segmentId)
+ }
+
test("test data loading CSV file") {
val testData = currentDirectory + "/src/test/resources/sample.csv"
sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
@@ -99,6 +122,23 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test data loading into table whose name has '_'") {
+ sql("DROP TABLE IF EXISTS load_test")
+ sql(""" CREATE TABLE load_test(id int, name string, city string, age int)
+ STORED BY 'org.apache.carbondata.format' """)
+ val testData = currentDirectory + "/src/test/resources/sample.csv"
+ try {
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table load_test")
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table load_test")
+ } catch {
+ case ex: Exception =>
+ assert(false)
+ }
+ assert(checkSegmentExists("0", "default", "load_test"))
+ assert(checkSegmentExists("1", "default", "load_test"))
+ sql("DROP TABLE load_test")
+ }
+
override def afterAll {
sql("DROP TABLE loadtest")
}
[2/2] incubator-carbondata git commit: [CARBONDATA-412] Fix load bug
when table name has '_' This closes #320
Posted by ja...@apache.org.
[CARBONDATA-412] Fix load bug when table name has '_' This closes #320
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ddeb0042
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ddeb0042
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ddeb0042
Branch: refs/heads/master
Commit: ddeb00425537ff59bdfba76779c5d96287e07d2e
Parents: a561d86 fc1d620
Author: jackylk <ja...@huawei.com>
Authored: Wed Dec 21 16:45:29 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Wed Dec 21 16:45:29 2016 +0800
----------------------------------------------------------------------
.../core/carbon/path/CarbonTablePath.java | 8 ++--
.../core/constants/CarbonCommonConstants.java | 4 ++
.../dataload/TestLoadDataGeneral.scala | 42 +++++++++++++++++++-
3 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------