You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/19 13:59:51 UTC
[1/2] carbondata git commit: delete carbon-spark-shell script
Repository: carbondata
Updated Branches:
refs/heads/master f8db66d6b -> f8a17fb2c
delete carbon-spark-shell script
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bf136130
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bf136130
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bf136130
Branch: refs/heads/master
Commit: bf13613041fe5763f73d2b1c4eb218e5b215d9e1
Parents: f8db66d
Author: chenerlu <ch...@huawei.com>
Authored: Mon Jun 19 15:53:40 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Jun 19 21:59:10 2017 +0800
----------------------------------------------------------------------
bin/carbon-spark-shell | 116 -------------------
.../apache/spark/repl/CarbonSparkILoop.scala | 72 ------------
.../org/apache/spark/repl/carbon/Main.scala | 33 ------
.../apache/spark/repl/CarbonSparkILoop.scala | 72 ------------
4 files changed, 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf136130/bin/carbon-spark-shell
----------------------------------------------------------------------
diff --git a/bin/carbon-spark-shell b/bin/carbon-spark-shell
deleted file mode 100755
index 78e350c..0000000
--- a/bin/carbon-spark-shell
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# Shell script for starting the Spark Shell REPL
-
-cygwin=false
-case "`uname`" in
- CYGWIN*) cygwin=true;;
-esac
-
-# Enter posix mode for bash
-set -o posix
-
-export FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
-export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]"
-
-export FWDIR=$SPARK_HOME
-export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
-ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.10"
-GREP_OPTIONS=
-num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc -l)"
-if [ "$num_jars" -eq "0" -a -z "$ASSEMBLY_DIR" ]; then
- echo "Failed to find Carbondata assembly in $ASSEMBLY_DIR." 1>&2
- echo "You need to build Carbondata before running this program." 1>&2
- exit 1
-fi
-ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" || true)"
-if [ "$num_jars" -gt "1" ]; then
- echo "Found multiple Carbondata assembly jars in $ASSEMBLY_DIR:" 1>&2
- echo "$ASSEMBLY_JARS" 1>&2
- echo "Please remove all but one jar." 1>&2
- exit 1
-fi
-
-ASSEMBLY_JAR="${ASSEMBLY_DIR}/${ASSEMBLY_JARS}"
-export JAR="$ASSEMBLY_JAR"
-export SPARK_CLASSPATH=$SPARK_CLASSPATH:$JAR
-export CARBON_HOME=$CARBON_SOURCE
-
-
-# SPARK-4161: scala does not assume use of the java classpath,
-# so we need to add the "-Dscala.usejavacp=true" flag manually. We
-# do this specifically for the Spark shell because the scala REPL
-# has its own class loader, and any additional classpath specified
-# through spark.driver.extraClassPath is not automatically propagated.
-SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dscala.usejavacp=true"
-
-function main() {
- if $cygwin; then
- # Workaround for issue involving JLine and Cygwin
- # (see http://sourceforge.net/p/jline/bugs/40/).
- # If you're using the Mintty terminal emulator in Cygwin, may need to set the
- # "Backspace sends ^H" setting in "Keys" section of the Mintty options
- # (see https://github.com/sbt/sbt/issues/562).
- stty -icanon min 1 -echo > /dev/null 2>&1
- export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix"
- "$FWDIR"/bin/spark-submit "$@" --name "Carbon Spark shell" --class org.apache.spark.repl.carbon.Main $JAR
- stty icanon echo > /dev/null 2>&1
- else
- export SPARK_SUBMIT_OPTS
- "$FWDIR"/bin/spark-submit "$@" --name "Carbon Spark shell" --class org.apache.spark.repl.carbon.Main $JAR
- fi
-}
-
-# Copy restore-TTY-on-exit functions from Scala script so spark-shell exits properly even in
-# binary distribution of Spark where Scala is not installed
-exit_status=127
-saved_stty=""
-
-# restore stty settings (echo in particular)
-function restoreSttySettings() {
- stty $saved_stty
- saved_stty=""
-}
-
-function onExit() {
- if [[ "$saved_stty" != "" ]]; then
- restoreSttySettings
- fi
- exit $exit_status
-}
-
-# to reenable echo if we are interrupted before completing.
-trap onExit INT
-
-# save terminal settings
-saved_stty=$(stty -g 2>/dev/null)
-# clear on error so we don't later try to restore them
-if [[ ! $? ]]; then
- saved_stty=""
-fi
-
-main "$@"
-
-# record the exit status lest it be overwritten:
-# then reenable echo and propagate the code.
-exit_status=$?
-onExit
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf136130/integration/spark/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala b/integration/spark/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
deleted file mode 100644
index cee026d..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-class CarbonSparkILoop extends SparkILoop {
-
- override def initializeSpark() {
- intp.beQuietDuring {
- command("""
- if(org.apache.spark.repl.carbon.Main.interp == null) {
- org.apache.spark.repl.carbon.Main.main(Array[String]())
- }
- """)
- command("val i1 = org.apache.spark.repl.carbon.Main.interp")
- command("import i1._")
- command("""
- @transient val sc = {
- val _sc = i1.createSparkContext()
- println("Spark context available as sc.")
- _sc
- }
- """)
- command("import org.apache.spark.SparkContext._")
- command("import org.apache.spark.sql.CarbonContext")
- command("""
- @transient val cc = {
- val _cc = {
- import java.io.File
- val path = System.getenv("CARBON_HOME") + "/bin/carbonshellstore"
- val store = new File(path)
- store.mkdirs()
- val storePath = sc.getConf.getOption("spark.carbon.storepath")
- .getOrElse(store.getCanonicalPath)
- new CarbonContext(sc, storePath, store.getCanonicalPath)
- }
- println("Carbon context available as cc.")
- _cc
- }
- """)
-
- command("import org.apache.spark.sql.SQLContext")
- command("""
- @transient val sqlContext = {
- val _sqlContext = new SQLContext(sc)
- println("SQL context available as sqlContext.")
- _sqlContext
- }
- """)
- command("import sqlContext.implicits._")
- command("import sqlContext.sql")
-
- command("import cc.implicits._")
- command("import cc.sql")
- command("import org.apache.spark.sql.functions._")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf136130/integration/spark/src/main/scala/org/apache/spark/repl/carbon/Main.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/repl/carbon/Main.scala b/integration/spark/src/main/scala/org/apache/spark/repl/carbon/Main.scala
deleted file mode 100644
index d4ab362..0000000
--- a/integration/spark/src/main/scala/org/apache/spark/repl/carbon/Main.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl.carbon
-
-import org.apache.spark.repl.{CarbonSparkILoop, SparkILoop}
-
-object Main {
- private var _interp: SparkILoop = _
-
- def interp: SparkILoop = _interp
-
- def interp_=(i: SparkILoop) { _interp = i }
-
- def main(args: Array[String]) {
- _interp = new CarbonSparkILoop
- _interp.process(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bf136130/integration/spark2/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala b/integration/spark2/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
deleted file mode 100644
index cee026d..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-class CarbonSparkILoop extends SparkILoop {
-
- override def initializeSpark() {
- intp.beQuietDuring {
- command("""
- if(org.apache.spark.repl.carbon.Main.interp == null) {
- org.apache.spark.repl.carbon.Main.main(Array[String]())
- }
- """)
- command("val i1 = org.apache.spark.repl.carbon.Main.interp")
- command("import i1._")
- command("""
- @transient val sc = {
- val _sc = i1.createSparkContext()
- println("Spark context available as sc.")
- _sc
- }
- """)
- command("import org.apache.spark.SparkContext._")
- command("import org.apache.spark.sql.CarbonContext")
- command("""
- @transient val cc = {
- val _cc = {
- import java.io.File
- val path = System.getenv("CARBON_HOME") + "/bin/carbonshellstore"
- val store = new File(path)
- store.mkdirs()
- val storePath = sc.getConf.getOption("spark.carbon.storepath")
- .getOrElse(store.getCanonicalPath)
- new CarbonContext(sc, storePath, store.getCanonicalPath)
- }
- println("Carbon context available as cc.")
- _cc
- }
- """)
-
- command("import org.apache.spark.sql.SQLContext")
- command("""
- @transient val sqlContext = {
- val _sqlContext = new SQLContext(sc)
- println("SQL context available as sqlContext.")
- _sqlContext
- }
- """)
- command("import sqlContext.implicits._")
- command("import sqlContext.sql")
-
- command("import cc.implicits._")
- command("import cc.sql")
- command("import org.apache.spark.sql.functions._")
- }
- }
-}
[2/2] carbondata git commit: [CARBONDATA-1191] Remove
carbon-spark-shell script This closes #1060
Posted by ja...@apache.org.
[CARBONDATA-1191] Remove carbon-spark-shell script This closes #1060
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f8a17fb2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f8a17fb2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f8a17fb2
Branch: refs/heads/master
Commit: f8a17fb2ce72eef94fe5fcca3c8a50df57c2fc6c
Parents: f8db66d bf13613
Author: jackylk <ja...@huawei.com>
Authored: Mon Jun 19 21:59:36 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Jun 19 21:59:36 2017 +0800
----------------------------------------------------------------------
bin/carbon-spark-shell | 116 -------------------
.../apache/spark/repl/CarbonSparkILoop.scala | 72 ------------
.../org/apache/spark/repl/carbon/Main.scala | 33 ------
.../apache/spark/repl/CarbonSparkILoop.scala | 72 ------------
4 files changed, 293 deletions(-)
----------------------------------------------------------------------