Repository: incubator-predictionio
Updated Branches:
refs/heads/develop cc4e2e0a2 -> a81302171
[PIO-72] Fix class loading for pio-shell
Closes #401
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d894b72a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d894b72a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d894b72a
Branch: refs/heads/develop
Commit: d894b72af8975aef872aae5311f3c7f75aa00961
Parents: cc4e2e0
Author: Chan Lee <ch...@gmail.com>
Authored: Thu Jul 27 14:54:37 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Jul 27 14:54:37 2017 -0700
----------------------------------------------------------------------
bin/compute-classpath.sh | 23 +++++++++-----------
bin/pio-class | 2 --
bin/pio-shell | 5 ++---
tests/build_docker.sh | 13 ++++++-----
.../org/apache/predictionio/tools/Common.scala | 19 +++++++++-------
5 files changed, 31 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 032d111..cd0527f 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -17,20 +17,11 @@
# limitations under the License.
#
-SCALA_VERSION=2.10
-
# Figure out where PredictionIO is installed
FWDIR="$(cd `dirname $0`/..; pwd)"
. ${FWDIR}/bin/load-pio-env.sh
-# Build up classpath
-CLASSPATH="${PIO_CONF_DIR}"
-
-CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*"
-
-ASSEMBLY_DIR="${FWDIR}/assembly"
-
if [ -n "$JAVA_HOME" ]; then
JAR_CMD="$JAVA_HOME/bin/jar"
else
@@ -41,12 +32,19 @@ fi
if [ -f "${FWDIR}/RELEASE" ]; then
assembly_folder="${FWDIR}"/lib
else
- assembly_folder="${ASSEMBLY_DIR}"
+ assembly_folder="${FWDIR}"/assembly/src/universal/lib
fi
-ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+DATA_JARS=$(ls "${assembly_folder}"/spark/pio-data-*assembly*.jar 2>/dev/null)
+# Comma-separated list of assembly jars for submitting to spark-shell
+ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -)
-CLASSPATH="$CLASSPATH:${ASSEMBLY_JAR}"
+# Build up classpath
+CLASSPATH="${PIO_CONF_DIR}"
+CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*"
+CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*"
+CLASSPATH="$CLASSPATH:${MAIN_JAR}"
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this
# assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
@@ -70,5 +68,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then
CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER"
fi
-
echo "$CLASSPATH"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-class
----------------------------------------------------------------------
diff --git a/bin/pio-class b/bin/pio-class
index 8bde407..94cc520 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -22,8 +22,6 @@ case "`uname`" in
CYGWIN*) cygwin=true;;
esac
-SCALA_VERSION=2.10
-
# Figure out where PredictionIO is installed
FWDIR="$(cd `dirname $0`/..; pwd)"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-shell
----------------------------------------------------------------------
diff --git a/bin/pio-shell b/bin/pio-shell
index d47008f..5a5745a 100755
--- a/bin/pio-shell
+++ b/bin/pio-shell
@@ -55,11 +55,10 @@ fi
if [[ "$1" == "--with-spark" ]]
then
echo "Starting the PIO shell with the Apache Spark Shell."
- # compute the $ASSEMPLY_JAR, the location of the assemply jar, with
- # bin/compute-classpath.sh
+ # Get paths of assembly jars to pass to spark-shell
. ${PIO_HOME}/bin/compute-classpath.sh
shift
- $SPARK_HOME/bin/spark-shell --jars $ASSEMBLY_JAR $@
+ ${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@
else
echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m"
echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit arguments...]'.\033[0m"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index d467e97..b17a304 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -38,11 +38,14 @@ fi
-Dhadoop.version=$PIO_HADOOP_VERSION \
-Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
sbt/sbt clean storage/clean
-rm -rf assembly/*.jar
-cp dist/lib/*.jar assembly/
-rm -rf lib/spark
-mkdir -p lib/spark
-cp dist/lib/spark/*.jar lib/spark
+
+assembly_folder=assembly/src/universal/lib
+rm -rf ${assembly_folder}/*.jar
+rm -rf ${assembly_folder}/spark
+mkdir -p ${assembly_folder}/spark
+
+cp dist/lib/*.jar ${assembly_folder}
+cp dist/lib/spark/*.jar ${assembly_folder}/spark
rm *.tar.gz
docker build -t predictionio/pio .
popd
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
index 0ddd644..047b8dd 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -65,6 +65,15 @@ object Common extends EitherLogging {
}
}
+ def getCoreDir(pioHome: String): String = {
+ if (new File(pioHome + File.separator + "RELEASE").exists) {
+ pioHome + File.separator + "lib"
+ } else {
+ Array(pioHome, "assembly", "src", "universal", "lib")
+ .mkString(File.separator)
+ }
+ }
+
def getEngineDirPath(directory: Option[String]): String = {
new File(directory.getOrElse(".")).getCanonicalPath
}
@@ -97,18 +106,12 @@ object Common extends EitherLogging {
def jarFilesAt(path: File): Array[File] = path.listFiles filter {
_.getName.toLowerCase.endsWith(".jar")
}
- jarFilesAt(new File(pioHome, "lib/spark"))
+ jarFilesAt(new File(getCoreDir(pioHome) + File.separator + "spark"))
}
def coreAssembly(pioHome: String): Expected[File] = {
val core = s"pio-assembly-${BuildInfo.version}.jar"
- val coreDir =
- if (new File(pioHome + File.separator + "RELEASE").exists) {
- new File(pioHome + File.separator + "lib")
- } else {
- new File(pioHome + File.separator + "assembly")
- }
- val coreFile = new File(coreDir, core)
+ val coreFile = new File(getCoreDir(pioHome), core)
if (coreFile.exists) {
Right(coreFile)
} else {
[PIO-72] Add JDBC JARs when they are present Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/a8130217 Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/a8130217 Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/a8130217 Branch: refs/heads/develop Commit: a813021714ae8930c0d7e2061a15ab897005d188 Parents: d894b72 Author: Donald Szeto <do...@apache.org> Authored: Fri Jul 28 11:08:00 2017 -0700 Committer: Donald Szeto <do...@apache.org> Committed: Fri Jul 28 11:08:00 2017 -0700 ---------------------------------------------------------------------- bin/compute-classpath.sh | 2 ++ 1 file changed, 2 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a8130217/bin/compute-classpath.sh ---------------------------------------------------------------------- diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index cd0527f..3e59ca7 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -63,9 +63,11 @@ if [ -n "$ES_CONF_DIR" ]; then fi if [ -n "$POSTGRES_JDBC_DRIVER" ]; then CLASSPATH="$CLASSPATH:$POSTGRES_JDBC_DRIVER" + ASSEMBLY_JARS="$ASSEMBLY_JARS,$POSTGRES_JDBC_DRIVER" fi if [ -n "$MYSQL_JDBC_DRIVER" ]; then CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER" + ASSEMBLY_JARS="$ASSEMBLY_JARS,$MYSQL_JDBC_DRIVER" fi echo "$CLASSPATH"