You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by sh...@apache.org on 2019/01/23 04:28:06 UTC
[predictionio] branch develop updated: [PIO-199] Spark 2.4 support
for Scala 2.11 (#503)
This is an automated email from the ASF dual-hosted git repository.
shimamoto pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/predictionio.git
The following commit(s) were added to refs/heads/develop by this push:
new 9f8be58 [PIO-199] Spark 2.4 support for Scala 2.11 (#503)
9f8be58 is described below
commit 9f8be581271cd163ea289a20720e7b0fb6487651
Author: takako shimamoto <ch...@gmail.com>
AuthorDate: Wed Jan 23 13:27:58 2019 +0900
[PIO-199] Spark 2.4 support for Scala 2.11 (#503)
* Spark 2.4 support for Scala 2.11
* Update Spark 2.2 series in .travis.yml to 2.2.3
---
.travis.yml | 9 +++--
build.sbt | 40 ++++++----------------
data/build.sbt | 1 +
.../source/install/install-sourcecode.html.md.erb | 28 +++++----------
storage/elasticsearch/build.sbt | 6 ++--
5 files changed, 29 insertions(+), 55 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 42662de..82501d4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -73,7 +73,7 @@ env:
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.2
+ PIO_SPARK_VERSION=2.2.3
PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
@@ -94,13 +94,18 @@ env:
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
PIO_SCALA_VERSION=2.11.12
- PIO_SPARK_VERSION=2.2.2
+ PIO_SPARK_VERSION=2.2.3
PIO_HADOOP_VERSION=2.7.7
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
PIO_SCALA_VERSION=2.11.12
PIO_SPARK_VERSION=2.3.2
PIO_HADOOP_VERSION=2.7.7
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.12
+ PIO_SPARK_VERSION=2.4.0
+ PIO_HADOOP_VERSION=2.7.7
- BUILD_TYPE=LicenseCheck
diff --git a/build.sbt b/build.sbt
index 65c4ca3..7f2bc09 100644
--- a/build.sbt
+++ b/build.sbt
@@ -16,25 +16,6 @@
*/
import PIOBuild._
-lazy val scalaSparkDepsVersion = Map(
- "2.11" -> Map(
- "2.0" -> Map(
- "akka" -> "2.5.16",
- "hadoop" -> "2.7.7",
- "json4s" -> "3.2.11"),
- "2.1" -> Map(
- "akka" -> "2.5.17",
- "hadoop" -> "2.7.7",
- "json4s" -> "3.2.11"),
- "2.2" -> Map(
- "akka" -> "2.5.17",
- "hadoop" -> "2.7.7",
- "json4s" -> "3.2.11"),
- "2.3" -> Map(
- "akka" -> "2.5.17",
- "hadoop" -> "2.7.7",
- "json4s" -> "3.2.11")))
-
name := "apache-predictionio-parent"
version in ThisBuild := "0.14.0-SNAPSHOT"
@@ -60,23 +41,22 @@ sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.3")
sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
-akkaVersion in ThisBuild := sys.props.getOrElse(
- "akka.version",
- scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("akka"))
+hadoopVersion in ThisBuild := sys.props.getOrElse("hadoop.version", "2.7.7")
+
+akkaVersion in ThisBuild := sys.props.getOrElse("akka.version", "2.5.17")
lazy val es = sys.props.getOrElse("elasticsearch.version", "5.6.9")
elasticsearchVersion in ThisBuild := es
-lazy val hbase = sys.props.getOrElse("hbase.version", "1.2.6")
-
-hbaseVersion in ThisBuild := hbase
+hbaseVersion in ThisBuild := sys.props.getOrElse("hbase.version", "1.2.6")
-json4sVersion in ThisBuild := scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("json4s")
-
-hadoopVersion in ThisBuild := sys.props.getOrElse(
- "hadoop.version",
- scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("hadoop"))
+json4sVersion in ThisBuild := {
+ sparkBinaryVersion.value match {
+ case "2.0" | "2.1" | "2.2" | "2.3" => "3.2.11"
+ case "2.4" => "3.5.3"
+ }
+}
val conf = file("conf")
diff --git a/data/build.sbt b/data/build.sbt
index 23dff1d..6592536 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -20,6 +20,7 @@ import PIOBuild._
name := "apache-predictionio-data"
libraryDependencies ++= Seq(
+ "org.scala-lang" % "scala-reflect" % scalaVersion.value,
"com.github.nscala-time" %% "nscala-time" % "2.6.0",
"com.google.guava" % "guava" % "14.0.1",
"com.typesafe.akka" %% "akka-http-testkit" % "10.1.5" % "test",
diff --git a/docs/manual/source/install/install-sourcecode.html.md.erb b/docs/manual/source/install/install-sourcecode.html.md.erb
index 9a1e456..fe5de3d 100644
--- a/docs/manual/source/install/install-sourcecode.html.md.erb
+++ b/docs/manual/source/install/install-sourcecode.html.md.erb
@@ -27,8 +27,8 @@ replace `/home/abc` with your own home directory wherever you see it.
You can use pre-built binary distribution for Apache PredictionIO® if you are
building against
-* Scala 2.11.8
-* Spark 2.1.1
+* Scala 2.11.12
+* Spark 2.1.3
* Hadoop 2.7.7
* Elasticsearch 5.6.9
@@ -100,17 +100,14 @@ and complete information.
### Building
Run the following at the directory where you downloaded the source code to build
-Apache PredictionIO®. By default, the build will be against
-
-* Scala 2.11.8
-* Spark 2.1.1
-* Hadoop 2.7.7
-* Elasticsearch 5.6.9
+Apache PredictionIO®.
+As an example, if you want to build PredictionIO to support Scala 2.11.12,
+Spark 2.4.0, and Elasticsearch 6.4.2, you can do
```
$ tar zxvf apache-predictionio-<%= data.versions.pio %>.tar.gz
$ cd apache-predictionio-<%= data.versions.pio %>
-$ ./make-distribution.sh
+$ ./make-distribution.sh -Dscala.version=2.11.12 -Dspark.version=2.4.0 -Delasticsearch.version=6.4.2
```
You should see something like the following when it finishes building
@@ -136,17 +133,10 @@ Starting from version 0.11.0, PredictionIO can be built against different
versions of dependencies. As of writing, one could build PredictionIO against
these different dependencies:
-* 2.11.x
-* 2.0.x, 2.1.x, 2.2.x, 2.3.x
+* Scala 2.11.x
+* Spark 2.0.x, 2.1.x, 2.2.x, 2.3.x, 2.4.x
* Hadoop 2.6.x, 2.7.x
-* Elasticsearch 1.7.x(deprecated), 5.x
-
-As an example, if you want to build PredictionIO to support Scala 2.11.8,
-Spark 2.1.0, and Elasticsearch 5.3.0, you can do
-
-```
-$ ./make-distribution.sh -Dscala.version=2.11.8 -Dspark.version=2.1.0 -Delasticsearch.version=5.3.0
-```
+* Elasticsearch 1.7.x(deprecated), 5.6.x, 6.x
## Installing Dependencies
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index f340136..3dbd3de 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -19,15 +19,13 @@ import PIOBuild._
name := "apache-predictionio-data-elasticsearch"
-elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13")
-
elasticsearchVersion := (if (majorVersion(elasticsearchVersion.value) < 5) "5.6.9" else elasticsearchVersion.value)
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
- "org.elasticsearch.client" % "elasticsearch-rest-client" % elasticsearchVersion.value,
- "org.elasticsearch" %% elasticsearchSparkArtifact.value % elasticsearchVersion.value
+ "org.elasticsearch.client" % "elasticsearch-rest-client" % elasticsearchVersion.value,
+ "org.elasticsearch" %% "elasticsearch-spark-20" % elasticsearchVersion.value
exclude("org.apache.spark", "*"),
"org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value,
"org.specs2" %% "specs2" % "2.3.13" % "test")