You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2017/03/16 14:52:17 UTC
[4/7] incubator-predictionio git commit: Grand unification
Grand unification
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/be8397fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/be8397fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/be8397fc
Branch: refs/heads/feature/xbuild
Commit: be8397fc63a967a5cabf838c52096bd806991db7
Parents: b6c90a0
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 16 00:08:15 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 16 00:08:15 2017 -0700
----------------------------------------------------------------------
build.sbt | 177 +++++++++----------
common/build.sbt | 2 +
conf/pio-vendors.sh | 26 +++
conf/set_build_profile.sh | 31 ----
core/build.sbt | 2 +
data/build.sbt | 2 +
e2/build.sbt | 2 +
make-distribution.sh | 35 +---
project/Build.scala | 57 ------
project/PIOBuild.scala | 33 ++++
storage/elasticsearch/build.sbt | 2 +
storage/elasticsearch1/build.sbt | 2 +
storage/hbase/build.sbt | 2 +
storage/hdfs/build.sbt | 2 +
storage/jdbc/build.sbt | 2 +
storage/localfs/build.sbt | 2 +
tests/Dockerfile | 7 +-
tests/Dockerfile-es1 | 57 ------
tests/after_script.travis.sh | 8 +-
tests/build_docker.sh | 38 ++--
tests/docker-compose-es1.yml | 38 ----
tests/docker-compose.yml | 2 +-
tests/docker-files/env-conf/pio-env-es1.sh | 103 -----------
tests/docker-files/env-conf/pio-env.sh | 10 +-
.../engines/recommendation-engine/build.sbt | 4 +-
.../engines/recommendation-engine/manifest.json | 1 -
.../engines/recommendation-engine/pio.sbt | 4 -
.../recommendation-engine/project/pio-build.sbt | 1 -
tests/run_docker.sh | 19 +-
tests/script.travis.sh | 2 +-
tests/unit.sh | 13 +-
tools/build.sbt | 1 +
32 files changed, 222 insertions(+), 465 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 330dd06..16ce10e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -15,55 +15,36 @@
* limitations under the License.
*/
+import PIOBuild._
import UnidocKeys._
-import scala.math.Ordering.Implicits._
-
-lazy val profiles: Map[String, Profile] =
- Map(
- "scala-2.10" -> Profile(
- name="scala-2.10",
- scalaVersion="2.10.5",
- sparkVersion="1.6.3",
- hadoopVersion="2.6.4",
- akkaVersion="2.3.15"),
-
- "scala-2.11" -> Profile(
- name="scala-2.11",
- scalaVersion="2.11.8",
- sparkVersion="2.0.2",
- hadoopVersion="2.7.3",
- akkaVersion="2.4.10"))
-
-lazy val defaultProfile = "scala-2.10"
-
-buildProfile := {
- val profileName = sys.props.get("build.profile").getOrElse(defaultProfile)
- val profile = profiles(profileName)
- val scalaVersion = profile.scalaVersion
-
- val sparkVersion = sys.props.get("spark.version") map { sv =>
- if ((versionMajor(sv), versionMinor(sv)) < (1, 6)) {
- throw new IllegalArgumentException("Spark versions below 1.6 are no longer supported")
- } else if (versionMajor(sv) >= 2 &&
- (versionMajor(scalaVersion), versionMinor(scalaVersion)) < (2, 11)) {
- throw new IllegalArgumentException("Spark 2.x requires Scala 2.11 and above")
- } else {
- sv
- }
- } getOrElse(profile.sparkVersion)
-
- val hadoopVersion = sys.props.get("hadoop.version").getOrElse(profile.hadoopVersion)
-
- if (hadoopVersion != profile.hadoopVersion || sparkVersion != profile.sparkVersion) {
- profile.copy(
- name = profile.name + "-custom",
- sparkVersion = sparkVersion,
- hadoopVersion = hadoopVersion)
- } else {
- profile
- }
-}
+lazy val scalaSparkDependenciesVersion = Map(
+ "2.10" -> Map(
+ "1.6" -> Map(
+ "akka" -> "2.3.15",
+ "hadoop" -> "2.6.5",
+ "json4s" -> "3.2.10"),
+ "2.0" -> Map(
+ "akka" -> "2.3.16",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.1" -> Map(
+ "akka" -> "2.3.16",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11")),
+ "2.11" -> Map(
+ "1.6" -> Map(
+ "akka" -> "2.3.15",
+ "hadoop" -> "2.6.5",
+ "json4s" -> "3.2.10"),
+ "2.0" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.1" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11")))
name := "apache-predictionio-parent"
@@ -71,13 +52,9 @@ version in ThisBuild := "0.11.0-SNAPSHOT"
organization in ThisBuild := "org.apache.predictionio"
-scalaVersion in ThisBuild := {
- val version = buildProfile.value.scalaVersion
- if (versionMinor(version) < 11) {
- sLog.value.warn(s"Scala version ${version} is deprecated!")
- }
- version
-}
+scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.10.6")
+
+crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
@@ -87,13 +64,22 @@ fork in (ThisBuild, run) := true
javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7",
"-Xlint:deprecation", "-Xlint:unchecked")
-akkaVersion in ThisBuild := buildProfile.value.akkaVersion
+// Ignore differentiation of Spark patch levels
+sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "1.6.3")
+
+akkaVersion in ThisBuild := sys.props.getOrElse(
+ "akka.version",
+ scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("akka"))
+
+lazy val es = sys.props.getOrElse("elasticsearch.version", "1.7.6")
-json4sVersion in ThisBuild := "3.2.10"
+elasticsearchVersion in ThisBuild := es
-sparkVersion in ThisBuild := buildProfile.value.sparkVersion
+json4sVersion in ThisBuild := scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("json4s")
-hadoopVersion in ThisBuild := buildProfile.value.hadoopVersion
+hadoopVersion in ThisBuild := sys.props.getOrElse(
+ "hadoop.version",
+ scalaSparkDependenciesVersion(versionPrefix(scalaVersion.value))(versionPrefix(sparkVersion.value))("hadoop"))
val pioBuildInfoSettings = buildInfoSettings ++ Seq(
sourceGenerators in Compile <+= buildInfo,
@@ -115,7 +101,7 @@ val genjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
// in unit tests of data subproject are used later in spark jobs executed in core.
// They need to have properly configured classloaders to load core classes for spark
// in subsequent tests.
-def coreClasses(baseDirectory: java.io.File, scalaVersion: String) = Seq(
+def coreClasses(baseDirectory: java.io.File, scalaVersion: String): Seq[File] = Seq(
baseDirectory / s"../core/target/scala-${versionPrefix(scalaVersion)}/classes",
baseDirectory / s"../core/target/scala-${versionPrefix(scalaVersion)}/test-classes")
@@ -127,45 +113,54 @@ val commonSettings = Seq(
val common = (project in file("common")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ disablePlugins(sbtassembly.AssemblyPlugin)
val data = (project in file("data")).
dependsOn(common).
settings(commonSettings: _*).
settings(genjavadocSettings: _*).
settings(unmanagedSourceDirectories in Compile +=
- sourceDirectory.value / s"main/spark-${versionMajor(sparkVersion.value)}")
+ sourceDirectory.value / s"main/spark-${versionMajor(sparkVersion.value)}").
+ disablePlugins(sbtassembly.AssemblyPlugin)
val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataElasticsearch = (project in file("storage/elasticsearch")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataHbase = (project in file("storage/hbase")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataHdfs = (project in file("storage/hdfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataJdbc = (project in file("storage/jdbc")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataLocalfs = (project in file("storage/localfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val core = (project in file("core")).
dependsOn(data).
settings(commonSettings: _*).
settings(genjavadocSettings: _*).
settings(pioBuildInfoSettings: _*).
- enablePlugins(SbtTwirl)
+ enablePlugins(SbtTwirl).
+ disablePlugins(sbtassembly.AssemblyPlugin)
val tools = (project in file("tools")).
dependsOn(core).
@@ -173,12 +168,28 @@ val tools = (project in file("tools")).
settings(commonSettings: _*).
settings(genjavadocSettings: _*).
enablePlugins(SbtTwirl).
- settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value))
+ settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)).
+ settings(publishArtifact := false)
val e2 = (project in file("e2")).
settings(commonSettings: _*).
settings(genjavadocSettings: _*).
- settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value))
+ settings(fullClasspath in Test ++= coreClasses(baseDirectory.value, scalaVersion.value)).
+ disablePlugins(sbtassembly.AssemblyPlugin)
+
+val dataEs = if (versionMajor(es) == 1) dataElasticsearch1 else dataElasticsearch
+
+val subprojects = Seq(
+ common,
+ core,
+ data,
+ dataEs,
+ dataHbase,
+ dataHdfs,
+ dataJdbc,
+ dataLocalfs,
+ tools,
+ e2)
val root = (project in file(".")).
settings(commonSettings: _*).
@@ -237,18 +248,8 @@ val root = (project in file(".")).
"docs/javadoc/javadoc-overview.html",
"-noqualifier",
"java.lang")).
- aggregate(
- common,
- core,
- data,
- dataElasticsearch1,
- dataElasticsearch,
- dataHbase,
- dataHdfs,
- dataJdbc,
- dataLocalfs,
- tools,
- e2)
+ aggregate(subprojects map Project.projectToRef: _*).
+ disablePlugins(sbtassembly.AssemblyPlugin)
val pioUnidoc = taskKey[Unit]("Builds PredictionIO ScalaDoc")
@@ -308,11 +309,9 @@ parallelExecution in Global := false
testOptions in Test += Tests.Argument("-oDF")
-printProfile := {
- val profile = buildProfile.value
- println(s"PIO_PROFILE_VERSION=${profile.name}")
- println(s"PIO_SCALA_VERSION=${profile.scalaVersion}")
- println(s"PIO_SPARK_VERSION=${profile.sparkVersion}")
- println(s"PIO_HADOOP_VERSION=${profile.hadoopVersion}")
- println(s"PIO_AKKA_VERSION=${profile.akkaVersion}")
+printBuildInfo := {
+ println(s"PIO_SCALA_VERSION=${scalaVersion.value}")
+ println(s"PIO_SPARK_VERSION=${sparkVersion.value}")
+ println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}")
+ println(s"PIO_HADOOP_VERSION=${hadoopVersion.value}")
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/common/build.sbt
----------------------------------------------------------------------
diff --git a/common/build.sbt b/common/build.sbt
index 47fba3b..5a0fba1 100644
--- a/common/build.sbt
+++ b/common/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-common"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/conf/pio-vendors.sh
----------------------------------------------------------------------
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
index 00e7783..830b576 100644
--- a/conf/pio-vendors.sh
+++ b/conf/pio-vendors.sh
@@ -19,6 +19,32 @@
# IMPORTANT: PIO_*_VERSION for dependencies must be set before envoking this script.
# `source conf/set_build_profile.sh $BUILD_PROFILE` to get the proper versions
+if [ -z "$PIO_SCALA_VERSION" ]; then
+ PIO_SCALA_VERSION="2.10.6"
+fi
+
+if [ -z "$PIO_SPARK_VERSION" ]; then
+ PIO_SPARK_VERSION="1.6.3"
+fi
+
+if [ -z "$PIO_HADOOP_VERSION" ]; then
+ PIO_HADOOP_VERSION="2.6.5"
+fi
+
+if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then
+ PIO_ELASTICSEARCH_VERSION="1.7.3"
+fi
+
+ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
+
+if [ "$ES_MAJOR" = "1" ]; then
+ export ES_IMAGE="elasticsearch"
+ export ES_TAG="1"
+else
+ export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
+ export ES_TAG="5.2.2"
+fi
+
PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar
PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/conf/set_build_profile.sh
----------------------------------------------------------------------
diff --git a/conf/set_build_profile.sh b/conf/set_build_profile.sh
deleted file mode 100755
index 397e824..0000000
--- a/conf/set_build_profile.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Sets version of profile dependencies from sbt configuration.
-# eg. Run `source ./set_build_profile.sh scala-2.11`
-
-set -e
-
-if [[ "$#" -ne 1 ]]; then
- echo "Usage: set-build-profile.sh <build-profile>"
- exit 1
-fi
-
-set -a
-eval `sbt/sbt --error 'set showSuccess := false' -Dbuild.profile=$1 printProfile | grep '.*_VERSION=.*'`
-set +a
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
index 584133c..2d2a381 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-core"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index 27eeb78..778a8f1 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/e2/build.sbt
----------------------------------------------------------------------
diff --git a/e2/build.sbt b/e2/build.sbt
index 50de5e5..f30536a 100644
--- a/e2/build.sbt
+++ b/e2/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-e2"
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 2000340..70ba4a4 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -21,15 +21,12 @@ set -e
usage ()
{
- echo "Usage: $0 [-h|--help] [--with-es=x]"
+ echo "Usage: $0 [-h|--help]"
echo ""
echo " -h|--help Show usage"
- echo ""
- echo " --with-es=1 Build distribution with Elasticsearch 1 support as default"
- echo " --with-es=5 Build distribution with Elasticsearch 5 support as default"
}
-ES_VERSION=1
+JAVA_PROPS=()
for i in "$@"
do
@@ -39,11 +36,8 @@ case $i in
shift
exit
;;
- --with-es=*)
- ES_VERSION="${i#*=}"
- shift
- ;;
-D*)
+ JAVA_PROPS+=("$i")
shift
;;
*)
@@ -53,14 +47,6 @@ case $i in
esac
done
-if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ]
-then
- echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m"
-else
- usage
- exit 1
-fi
-
FWDIR="$(cd `dirname $0`; pwd)"
DISTDIR="${FWDIR}/dist"
@@ -68,10 +54,10 @@ VERSION=$(grep ^version ${FWDIR}/build.sbt | grep ThisBuild | grep -o '".*"' | s
echo "Building binary distribution for PredictionIO $VERSION..."
+sbt/sbt printBuildInfo
+
cd ${FWDIR}
-sbt/sbt "$@" common/publishLocal data/publishLocal core/publishLocal e2/publishLocal \
-dataElasticsearch1/assembly dataElasticsearch/assembly dataHbase/assembly dataHdfs/assembly \
-dataJdbc/assembly dataLocalfs/assembly tools/assembly
+sbt/sbt "$JAVA_PROPS" publishLocal assembly
cd ${FWDIR}
rm -rf ${DISTDIR}
@@ -79,8 +65,8 @@ mkdir -p ${DISTDIR}/bin
mkdir -p ${DISTDIR}/conf
mkdir -p ${DISTDIR}/lib
mkdir -p ${DISTDIR}/lib/spark
-mkdir -p ${DISTDIR}/lib/extra
mkdir -p ${DISTDIR}/project
+
mkdir -p ${DISTDIR}/sbt
cp ${FWDIR}/bin/* ${DISTDIR}/bin || :
@@ -90,13 +76,6 @@ cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
-if [ "$ES_VERSION" = "5" ]
-then
- mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra
-else
- mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra
-fi
-
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
rm -f ${DISTDIR}/conf/pio-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
deleted file mode 100644
index cc37912..0000000
--- a/project/Build.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt._
-import Keys._
-
-case class Profile(
- name: String,
- scalaVersion: String,
- sparkVersion: String,
- hadoopVersion: String,
- akkaVersion: String)
-
-object PIOBuild extends Build {
- val elasticsearchVersion = SettingKey[String](
- "elasticsearch-version",
- "The version of Elasticsearch used for building.")
- val json4sVersion = SettingKey[String](
- "json4s-version",
- "The version of JSON4S used for building.")
- val sparkVersion = SettingKey[String](
- "spark-version",
- "The version of Apache Spark used for building.")
- val hadoopVersion = SettingKey[String](
- "hadoop-version",
- "The version of Apache Hadoop used for building")
- val akkaVersion = SettingKey[String](
- "akka-version",
- "The version of Akka used for building")
- val buildProfile = SettingKey[Profile](
- "build-profile",
- "The dependency profile used for the build")
- val childrenPomExtra = SettingKey[scala.xml.NodeSeq](
- "children-pom-extra",
- "Extra POM data for children projects.")
-
- def versionPrefix(versionString: String) =
- versionString.split('.').take(2).mkString(".")
- def versionMajor(versionString: String) = versionString.split('.')(0).toInt
- def versionMinor(versionString: String) = versionString.split('.')(1).toInt
-
- lazy val printProfile = taskKey[Unit]("Print settings for the chosen profile")
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/project/PIOBuild.scala
----------------------------------------------------------------------
diff --git a/project/PIOBuild.scala b/project/PIOBuild.scala
new file mode 100644
index 0000000..69ac189
--- /dev/null
+++ b/project/PIOBuild.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt._
+
+object PIOBuild {
+ val elasticsearchVersion = settingKey[String]("The version of Elasticsearch used for building")
+ val json4sVersion = settingKey[String]("The version of JSON4S used for building")
+ val sparkVersion = settingKey[String]("The version of Apache Spark used for building")
+ val hadoopVersion = settingKey[String]("The version of Apache Hadoop used for building")
+ val akkaVersion = settingKey[String]("The version of Akka used for building")
+ val childrenPomExtra = settingKey[scala.xml.NodeSeq]("Extra POM data for children projects")
+
+ def versionPrefix(versionString: String): String = versionString.split('.').take(2).mkString(".")
+ def versionMajor(versionString: String): Int = versionString.split('.')(0).toInt
+ def versionMinor(versionString: String): Int = versionString.split('.')(1).toInt
+
+ lazy val printBuildInfo = taskKey[Unit]("Print build information")
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 7d9b2ad..a7d3ab5 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-elasticsearch"
elasticsearchVersion := "5.2.1"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index dde7285..c78c75d 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-elasticsearch1"
elasticsearchVersion := "1.7.3"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 513e294..22b53d8 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-hbase"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 7ddc86c..2942e56 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-hdfs"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 2011722..905ed53 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-jdbc"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index f9e101d..90a61db 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -15,6 +15,8 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-localfs"
libraryDependencies ++= Seq(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index a04b79c..619bdf8 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -20,13 +20,15 @@ FROM predictionio/pio
ARG SPARK_ARCHIVE
ARG SPARK_DIR
ARG PGSQL_JAR
-ARG BUILD_PROFILE
ARG PIO_SCALA_VERSION
ARG PIO_SPARK_VERSION
+ARG PIO_ELASTICSEARCH_VERSION
-ENV BUILD_PROFILE=$BUILD_PROFILE
ENV PIO_SCALA_VERSION=$PIO_SCALA_VERSION
ENV PIO_SPARK_VERSION=$PIO_SPARK_VERSION
+ENV PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION
+
+ENV PGSQL_JAR=$PGSQL_JAR
# WORKAROUND: es-hadoop stops on RDD#take(1)
ADD docker-files/${SPARK_ARCHIVE} /vendors
@@ -35,7 +37,6 @@ ENV SPARK_HOME /vendors/${SPARK_DIR}
COPY docker-files/${PGSQL_JAR} /drivers/${PGSQL_JAR}
COPY docker-files/init.sh init.sh
-COPY docker-files/set_build_profile.sh set_build_profile.sh
COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh
COPY docker-files/pgpass /root/.pgpass
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/Dockerfile-es1
----------------------------------------------------------------------
diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1
deleted file mode 100644
index 73d4c36..0000000
--- a/tests/Dockerfile-es1
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM predictionio/pio
-
-ENV SPARK_VERSION 1.6.3
-ENV ELASTICSEARCH_VERSION 1.7
-ENV HBASE_VERSION 1.0.0
-
-ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
-# WORKAROUND: es-hadoop stops on RDD#take(1)
-RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
-ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
-
-COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
-COPY docker-files/init.sh init.sh
-COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
-COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh
-COPY docker-files/pgpass /root/.pgpass
-RUN chmod 600 /root/.pgpass
-
-# Python
-RUN pip install python-dateutil
-RUN pip install pytz
-
-# Default repositories setup
-ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
-ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL
-ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL
-
-# JVM settings
-ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M'
-
-# Expose relevant ports
-# pio engine
-EXPOSE 8000
-# eventserver
-EXPOSE 7070
-
-ENV SLEEP_TIME 30
-
-ENTRYPOINT ["/init.sh"]
-CMD 'bash'
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index 8cbe2c0..e9d5792 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -18,12 +18,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $DIR/../conf/pio-vendors.sh
+
# Print a summary of containers used
docker ps -a
# Clean up used containers
-if [ "$ES_VERSION" = "1" ]; then
- docker-compose -f $DIR/docker-compose-es1.yml down
-else
- docker-compose -f $DIR/docker-compose.yml down
-fi
+docker-compose -f $DIR/docker-compose.yml down
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index 13a6f6a..9b6c2c6 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -22,7 +22,6 @@ docker pull predictionio/pio-testing-base
pushd $DIR/..
-source conf/set_build_profile.sh ${BUILD_PROFILE}
source conf/pio-vendors.sh
if [ ! -f $DIR/docker-files/${PGSQL_JAR} ]; then
wget $PGSQL_DOWNLOAD
@@ -33,36 +32,23 @@ if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then
mv $SPARK_ARCHIVE $DIR/docker-files/
fi
-if [ -z "$ES_VERSION" ]; then
- ./make-distribution.sh -Dbuild.profile=${BUILD_PROFILE}
-else
- ./make-distribution.sh --with-es=$ES_VERSION -Dbuild.profile=${BUILD_PROFILE}
-fi
+./make-distribution.sh \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
sbt/sbt clean
mkdir assembly
cp dist/lib/*.jar assembly/
mkdir -p lib/spark
-if [ -e dist/lib/spark/*.jar ]; then
- cp dist/lib/spark/*.jar lib/spark
-fi
+cp dist/lib/spark/*.jar lib/spark
rm *.tar.gz
docker build -t predictionio/pio .
popd
-if [ "$ES_VERSION" = "1" ]; then
- docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR \
- --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \
- --build-arg SPARK_DIR=$SPARK_DIR \
- --build-arg PGSQL_JAR=$PGSQL_JAR \
- --build-arg BUILD_PROFILE=$BUILD_PROFILE \
- --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \
- --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION
-else
- docker build -t predictionio/pio-testing $DIR \
- --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \
- --build-arg SPARK_DIR=$SPARK_DIR \
- --build-arg PGSQL_JAR=$PGSQL_JAR \
- --build-arg BUILD_PROFILE=$BUILD_PROFILE \
- --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \
- --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION
-fi
+docker build -t predictionio/pio-testing $DIR \
+ --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \
+ --build-arg SPARK_DIR=$SPARK_DIR \
+ --build-arg PGSQL_JAR=$PGSQL_JAR \
+ --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \
+ --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-compose-es1.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml
deleted file mode 100644
index 381f41c..0000000
--- a/tests/docker-compose-es1.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "2"
-services:
- elasticsearch:
- image: elasticsearch:1
- environment:
- - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- hbase:
- image: harisekhon/hbase:1.0
- postgres:
- image: postgres:9
- environment:
- POSTGRES_USER: pio
- POSTGRES_PASSWORD: pio
- POSTGRES_INITDB_ARGS: --encoding=UTF8
- pio-testing:
- image: predictionio/pio-testing-es1:latest
- depends_on:
- - elasticsearch
- - hbase
- - postgres
- volumes:
- - ~/.ivy2:/root/.ivy2
- - ~/.sbt:/root/.sbt
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 06126a5..ac1d91d 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,7 @@
version: "2"
services:
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+ image: ${ES_IMAGE}:${ES_TAG}
environment:
- xpack.security.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-files/env-conf/pio-env-es1.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh
deleted file mode 100644
index e1076ba..0000000
--- a/tests/docker-files/env-conf/pio-env-es1.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copy this file as pio-env.sh and edit it for your site's configuration.
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# PredictionIO Main Configuration
-#
-# This section controls core behavior of PredictionIO. It is very likely that
-# you need to change these to fit your site.
-
-# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$SPARK_HOME
-
-POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
-MYSQL_JDBC_DRIVER=
-
-# ES_CONF_DIR: You must configure this if you have advanced configuration for
-# your Elasticsearch setup.
-# ES_CONF_DIR=/opt/elasticsearch
-
-# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO
-# with Hadoop 2.
-# HADOOP_CONF_DIR=/opt/hadoop
-
-# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO
-# with HBase on a remote cluster.
-HBASE_CONF_DIR=$PIO_HOME/conf
-
-# Filesystem paths where PredictionIO uses as block storage.
-PIO_FS_BASEDIR=$HOME/.pio_store
-PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines
-PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp
-
-# PredictionIO Storage Configuration
-#
-# This section controls programs that make use of PredictionIO's built-in
-# storage facilities. Default values are shown below.
-#
-# For more information on storage configuration please refer to
-# https://docs.prediction.io/system/anotherdatastore/
-
-# Storage Repositories
-
-# Default is to use PostgreSQL
-PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta
-PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
-
-PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event
-PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
-
-PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model
-PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
-
-# Storage Data Sources
-
-# PostgreSQL Default Settings
-# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL
-# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and
-# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly
-PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
-PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio
-PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio
-PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
-
-# MySQL Example
-# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc
-# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio
-# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio
-# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio
-
-# Elasticsearch Example
-PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
-#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
-PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
-PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
-#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
-
-# Local File System Example
-PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
-PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models
-
-# HBase Example
-PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
-#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME
-
-# HDFS config
-PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs
-PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/docker-files/env-conf/pio-env.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh
index 0acf3a7..51d2e05 100644
--- a/tests/docker-files/env-conf/pio-env.sh
+++ b/tests/docker-files/env-conf/pio-env.sh
@@ -24,9 +24,9 @@
# you need to change these to fit your site.
# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$SPARK_HOME
+# SPARK_HOME=$SPARK_HOME
-POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
+POSTGRES_JDBC_DRIVER=/drivers/$PGSQL_JAR
MYSQL_JDBC_DRIVER=
# ES_CONF_DIR: You must configure this if you have advanced configuration for
@@ -88,7 +88,11 @@ PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
-PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
+if [ "$PIO_ELASTICSEARCH_VERSION" == "1" ]; then
+ PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
+else
+ PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
+fi
#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
# Local File System Example
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt
index dcb9fb6..13fe354 100644
--- a/tests/pio_tests/engines/recommendation-engine/build.sbt
+++ b/tests/pio_tests/engines/recommendation-engine/build.sbt
@@ -27,5 +27,5 @@ organization := "org.apache.predictionio"
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-SNAPSHOT" % "provided",
- "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.2") % "provided",
- "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.2") % "provided")
+ "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided",
+ "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/manifest.json
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/manifest.json b/tests/pio_tests/engines/recommendation-engine/manifest.json
deleted file mode 100644
index 9c545ce..0000000
--- a/tests/pio_tests/engines/recommendation-engine/manifest.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"yDBpzjz39AjIxlOAh8W4t3QSc75uPCuJ","version":"ee98ff9c009ef0d9fb1bc6b78750b83a0ceb37b2","name":"recommendation-engine","description":"pio-autogen-manifest","files":[],"engineFactory":""}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/pio.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/pio.sbt b/tests/pio_tests/engines/recommendation-engine/pio.sbt
deleted file mode 100644
index 5d8c366..0000000
--- a/tests/pio_tests/engines/recommendation-engine/pio.sbt
+++ /dev/null
@@ -1,4 +0,0 @@
-// Generated automatically by pio build.
-// Changes in this file will be overridden.
-
-pioVersion := "0.11.0-SNAPSHOT"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt b/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
deleted file mode 100644
index 8346a96..0000000
--- a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("io.prediction" % "pio-build" % "0.9.0")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index 6257fa2..ad7e189 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -46,17 +46,10 @@ shift
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ "$ES_VERSION" = "1" ]; then
- docker-compose -f $DIR/docker-compose-es1.yml run \
- -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
- -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
- -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
- pio-testing $@
-else
- docker-compose -f $DIR/docker-compose.yml run \
- -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
- -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
- -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
- pio-testing $@
-fi
+source $DIR/../conf/pio-vendors.sh
+docker-compose -f $DIR/docker-compose.yml run \
+ -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+ -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+ -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+ pio-testing $@
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/script.travis.sh b/tests/script.travis.sh
index efdbb9e..1d4985e 100755
--- a/tests/script.travis.sh
+++ b/tests/script.travis.sh
@@ -18,7 +18,7 @@
if [[ $BUILD_TYPE == Unit ]]; then
tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
- "/PredictionIO/tests/unit.sh $BUILD_PROFILE"
+ "/PredictionIO/tests/unit.sh"
else
tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
"python3 /PredictionIO/tests/pio_tests/tests.py"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tests/unit.sh
----------------------------------------------------------------------
diff --git a/tests/unit.sh b/tests/unit.sh
index 724b878..d6572c2 100755
--- a/tests/unit.sh
+++ b/tests/unit.sh
@@ -25,11 +25,20 @@ pushd /PredictionIO
set -a
source ./conf/pio-env.sh
set +a
+source ./conf/pio-vendors.sh
# Run stylecheck
-sbt/sbt -Dbuild.profile=$1 scalastyle
+sbt/sbt scalastyle \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
# Run all unit tests
-sbt/sbt -Dbuild.profile=$1 test
+sbt/sbt test \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
popd
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/be8397fc/tools/build.sbt
----------------------------------------------------------------------
diff --git a/tools/build.sbt b/tools/build.sbt
index d1ce2b9..0971b62 100644
--- a/tools/build.sbt
+++ b/tools/build.sbt
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+import PIOBuild._
import sbtassembly.AssemblyPlugin.autoImport._
name := "apache-predictionio-tools"