You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/03/08 11:28:30 UTC
[1/2] incubator-carbondata git commit: add spark-sql support for
spark2.1 carbon integration
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 8e8bc5de2 -> 0b8aaa609
add spark-sql support for spark2.1 carbon integration
modify CarbonSource
set master
fix comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a539f260
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a539f260
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a539f260
Branch: refs/heads/master
Commit: a539f260d447681e5f6a8a8eb28fc3552acf2c41
Parents: 8e8bc5d
Author: jackylk <ja...@huawei.com>
Authored: Sun Mar 5 20:24:34 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 8 16:56:49 2017 +0530
----------------------------------------------------------------------
bin/carbon-spark-sql | 8 ++-
.../org/apache/spark/sql/CarbonSource.scala | 3 +-
.../spark/sql/hive/cli/CarbonSQLCLIDriver.scala | 64 ++++++++++++++++++++
3 files changed, 72 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a539f260/bin/carbon-spark-sql
----------------------------------------------------------------------
diff --git a/bin/carbon-spark-sql b/bin/carbon-spark-sql
index 56aaebc..2c799d1 100755
--- a/bin/carbon-spark-sql
+++ b/bin/carbon-spark-sql
@@ -22,7 +22,6 @@
# Enter posix mode for bash
set -o posix
-
export CLASS="org.apache.spark.sql.hive.cli.CarbonSQLCLIDriver"
# Figure out where Spark is installed
@@ -33,7 +32,12 @@ fi
export FWDIR=$SPARK_HOME
export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
+
ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.10"
+if [ -d "$CARBON_SOURCE/assembly/target/scala-2.11" ]; then
+ ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
+fi
+
GREP_OPTIONS=
num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc -l)"
if [ "$num_jars" -eq "0" -a -z "$ASSEMBLY_DIR" ]; then
@@ -76,7 +80,6 @@ fi
#split options and cli_options, final submit example is spark-submit options --class xxx xxx.jar cli_options
options=""
-cmd
while true ; do
case "$1" in
-d|--define|-f|-i|--hiveconf|--hivevar|--database|-e|-S|--silent) cmd="$1"; break;;
@@ -86,4 +89,5 @@ while true ; do
esac
done
+echo "$FWDIR"/bin/spark-submit $options --class "$CLASS" "$JAR" "$@"
exec "$FWDIR"/bin/spark-submit $options --class "$CLASS" "$JAR" "$@"
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a539f260/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index f92c007..e28ee8f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -104,6 +104,7 @@ class CarbonSource extends CreatableRelationProvider
}
}
+
private def createTableIfNotExists(sparkSession: SparkSession, parameters: Map[String, String],
dataSchema: StructType): String = {
@@ -152,7 +153,7 @@ class CarbonSource extends CreatableRelationProvider
None
}
- val cm = TableCreator.prepareTableModel(false, Option(dbName),
+ val cm = TableCreator.prepareTableModel(ifNotExistPresent = false, Option(dbName),
tableName, fields, Nil, bucketFields, map)
CreateTable(cm, false).run(sparkSession)
CarbonEnv.get.carbonMetastore.storePath + s"/$dbName/$tableName"
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a539f260/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
new file mode 100644
index 0000000..22c7218
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/cli/CarbonSQLCLIDriver.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.cli
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.sql.{SparkSession, SQLContext}
+import org.apache.spark.sql.hive.thriftserver.{SparkSQLCLIDriver, SparkSQLEnv}
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+
+object CarbonSQLCLIDriver {
+
+ private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ var hiveContext: SQLContext = _
+ var sparkContext: SparkContext = _
+
+ def main(args: Array[String]): Unit = {
+ init()
+ SparkSQLEnv.sparkContext = sparkContext
+ SparkSQLEnv.sqlContext = hiveContext
+ SparkSQLCLIDriver.installSignalHandler()
+ SparkSQLCLIDriver.main(args)
+ }
+
+ def init() {
+ if (hiveContext == null) {
+ val sparkConf = new SparkConf(loadDefaults = true)
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val storePath = System.getenv("CARBON_HOME") + "/bin/carbonsqlclistore"
+ val warehouse = System.getenv("CARBON_HOME") + "/warehouse"
+ val carbon = SparkSession
+ .builder()
+ .master(System.getProperty("spark.master"))
+ .appName("CarbonSQLCLIDriver")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storePath, storePath)
+
+ hiveContext = carbon.sqlContext
+ hiveContext.conf.getAllConfs.toSeq.sorted.foreach { case (k, v) =>
+ LOGGER.debug(s"HiveConf var: $k=$v")
+ }
+ sparkContext = carbon.sqlContext.sparkContext
+
+ }
+ }
+
+}
[2/2] incubator-carbondata git commit: [CARBONDATA-746] Add spark-sql
CLI support for spark2.1 carbon integration This closes #623
Posted by ra...@apache.org.
[CARBONDATA-746] Add spark-sql CLI support for spark2.1 carbon integration This closes #623
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0b8aaa60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0b8aaa60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0b8aaa60
Branch: refs/heads/master
Commit: 0b8aaa609f732988ffc57072da82ec4045307966
Parents: 8e8bc5d a539f26
Author: ravipesala <ra...@gmail.com>
Authored: Wed Mar 8 16:57:45 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 8 16:57:45 2017 +0530
----------------------------------------------------------------------
bin/carbon-spark-sql | 8 ++-
.../org/apache/spark/sql/CarbonSource.scala | 3 +-
.../spark/sql/hive/cli/CarbonSQLCLIDriver.scala | 64 ++++++++++++++++++++
3 files changed, 72 insertions(+), 3 deletions(-)
----------------------------------------------------------------------