You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/07/09 16:55:33 UTC
[1/2] incubator-carbondata git commit: Fixed dataframe api
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 8f24031ce -> 367b9b2d2
Fixed dataframe api
Fixed relative path issue of metastore location
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/680e9c5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/680e9c5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/680e9c5f
Branch: refs/heads/master
Commit: 680e9c5f08f57f26606a471d1fb0a83f1ef9177a
Parents: 8f24031
Author: ravipesala <ra...@gmail.com>
Authored: Fri Jul 8 18:54:36 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Sat Jul 9 22:22:37 2016 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonContext.scala | 5 ++--
.../spark/sql/CarbonDatasourceRelation.scala | 27 ++++++++++----------
.../scala/org/carbondata/spark/package.scala | 2 +-
3 files changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/680e9c5f/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
index e88dcfe..44f5be7 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
@@ -83,11 +83,12 @@ class CarbonContext(
override protected def configure(): Map[String, String] = {
sc.hadoopConfiguration.addResource("hive-site.xml")
if (sc.hadoopConfiguration.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
- val hiveMetaStoreDB = metaStorePath + "/metastore_db"
+ val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
+ val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
logDebug(s"metastore db is going to be created in location : $hiveMetaStoreDB")
super.configure() ++ Map((CarbonCommonConstants.HIVE_CONNECTION_URL,
s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"),
- ("hive.metastore.warehouse.dir", metaStorePath + "/hivemetadata"))
+ ("hive.metastore.warehouse.dir", metaStorePathAbsolute + "/hivemetadata"))
} else {
super.configure()
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/680e9c5f/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 4d39eb2..2412e56 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -52,20 +52,19 @@ class CarbonSource
override def createRelation(
sqlContext: SQLContext,
parameters: Map[String, String]): BaseRelation = {
- if (parameters.get("tablePath") != None) {
- val options = new CarbonOption(parameters)
- val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
- val ident = tableIdentifier match {
- case Seq(name) => TableIdentifier(name, None)
- case Seq(db, name) => TableIdentifier(name, Some(db))
- }
- CarbonDatasourceRelation(ident, None)(sqlContext)
- } else if (parameters.get("path") != None) {
- CarbonDatasourceHadoopRelation(sqlContext, Array(parameters.get("path").get), parameters)
- } else {
- sys.error("Carbon table path not found")
+ // if path is provided we can directly create Hadoop relation. \
+ // Otherwise create datasource relation
+ parameters.get("path") match {
+ case Some(path) => CarbonDatasourceHadoopRelation(sqlContext, Array(path), parameters)
+ case _ =>
+ val options = new CarbonOption(parameters)
+ val tableIdentifier = options.tableIdentifier.split("""\.""").toSeq
+ val identifier = tableIdentifier match {
+ case Seq(name) => TableIdentifier(name, None)
+ case Seq(db, name) => TableIdentifier(name, Some(db))
+ }
+ CarbonDatasourceRelation(identifier, None)(sqlContext)
}
-
}
override def createRelation(
@@ -93,7 +92,7 @@ class CarbonSource
sys.error(s"ErrorIfExists mode, path $storePath already exists.")
case (SaveMode.Overwrite, true) =>
val cc = CarbonContext.getInstance(sqlContext.sparkContext)
- cc.sql(s"DROP CUBE IF EXISTS ${ options.dbName }.${ options.tableName }")
+ cc.sql(s"DROP TABLE IF EXISTS ${ options.dbName }.${ options.tableName }")
(true, false)
case (SaveMode.Overwrite, false) | (SaveMode.ErrorIfExists, false) =>
(true, false)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/680e9c5f/integration/spark/src/main/scala/org/carbondata/spark/package.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/package.scala b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
index 60c8880..282cb84 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/package.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
@@ -63,7 +63,7 @@ package object spark {
if (f.getPath.getName.startsWith("part-")) {
val newPath = s"${ f.getPath.getParent }/${ f.getPath.getName }.csv"
if (!fs.rename(f.getPath, new Path(newPath))) {
- cc.sql(s"DROP CUBE ${ options.tableName }")
+ cc.sql(s"DROP TABLE ${ options.tableName }")
throw new RuntimeException("File system rename failed when loading data into carbon")
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-46]Fixed dataframe
api in carbon This closes #31
Posted by ch...@apache.org.
[CARBONDATA-46]Fixed dataframe api in carbon This closes #31
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/367b9b2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/367b9b2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/367b9b2d
Branch: refs/heads/master
Commit: 367b9b2d26922de3ce48d8bef86f3825e217e871
Parents: 8f24031 680e9c5
Author: chenliang613 <ch...@apache.org>
Authored: Sat Jul 9 22:25:18 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Sat Jul 9 22:25:18 2016 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonContext.scala | 5 ++--
.../spark/sql/CarbonDatasourceRelation.scala | 27 ++++++++++----------
.../scala/org/carbondata/spark/package.scala | 2 +-
3 files changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------