You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2018/09/26 18:27:25 UTC
predictionio git commit: [PIO-171] Drop Scala 2.10 and Spark 1.6
support
Repository: predictionio
Updated Branches:
refs/heads/develop e48e93bed -> 718ebe2e8
[PIO-171] Drop Scala 2.10 and Spark 1.6 support
Closes #471
Project: http://git-wip-us.apache.org/repos/asf/predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/predictionio/commit/718ebe2e
Tree: http://git-wip-us.apache.org/repos/asf/predictionio/tree/718ebe2e
Diff: http://git-wip-us.apache.org/repos/asf/predictionio/diff/718ebe2e
Branch: refs/heads/develop
Commit: 718ebe2e8e98c5462d14ad2663c65288f0409972
Parents: e48e93b
Author: Naoki Takezoe <ta...@apache.org>
Authored: Wed Sep 26 11:25:01 2018 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Wed Sep 26 11:25:30 2018 -0700
----------------------------------------------------------------------
.travis.yml | 36 +++++++++++++-------
bin/pio-class | 2 +-
build.sbt | 21 ++----------
.../data/SparkVersionDependent.scala | 30 ----------------
docs/manual/source/install/index.html.md.erb | 6 ++--
.../tools/commands/Management.scala | 2 +-
6 files changed, 31 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 49cbb65..bdceb7e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -47,32 +47,44 @@ env:
matrix:
- BUILD_TYPE=Unit
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
PIO_ELASTICSEARCH_VERSION=1.7.3
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Integration
METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
- PIO_SCALA_VERSION=2.10.6
- PIO_SPARK_VERSION=1.6.3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
PIO_ELASTICSEARCH_VERSION=1.7.3
+ PIO_HADOOP_VERSION=2.6.5
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=ELASTICSEARCH MODELDATA_REP=S3
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.0.2
+ PIO_ELASTICSEARCH_VERSION=5.5.2
+ PIO_HADOOP_VERSION=2.6.5
- BUILD_TYPE=Unit
METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/bin/pio-class
----------------------------------------------------------------------
diff --git a/bin/pio-class b/bin/pio-class
index 9e01a40..67ae562 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -44,7 +44,7 @@ fi
# Make sure the Apache Spark version meets the prerequisite if it is a binary
# distribution
-MIN_SPARK_VERSION="1.6.3"
+MIN_SPARK_VERSION="2.0.2"
if [ -z "$SPARK_HOME" ]; then
echo -e "\033[0;31mSPARK_HOME must be set in conf/pio-env.sh, or in the environment!\033[0m"
exit 1
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index a067a3b..6a61521 100644
--- a/build.sbt
+++ b/build.sbt
@@ -17,24 +17,7 @@
import PIOBuild._
lazy val scalaSparkDepsVersion = Map(
- "2.10" -> Map(
- "1.6" -> Map(
- "akka" -> "2.3.15",
- "hadoop" -> "2.6.5",
- "json4s" -> "3.2.10"),
- "2.0" -> Map(
- "akka" -> "2.3.16",
- "hadoop" -> "2.7.3",
- "json4s" -> "3.2.11"),
- "2.1" -> Map(
- "akka" -> "2.3.16",
- "hadoop" -> "2.7.3",
- "json4s" -> "3.2.11")),
"2.11" -> Map(
- "1.6" -> Map(
- "akka" -> "2.3.15",
- "hadoop" -> "2.6.5",
- "json4s" -> "3.2.10"),
"2.0" -> Map(
"akka" -> "2.4.17",
"hadoop" -> "2.7.3",
@@ -62,7 +45,7 @@ scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.11.8")
scalaBinaryVersion in ThisBuild := binaryVersion(scalaVersion.value)
-crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
+crossScalaVersions in ThisBuild := Seq("2.11.8")
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
@@ -73,7 +56,7 @@ javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.8", "-target", "1.8",
"-Xlint:deprecation", "-Xlint:unchecked")
// Ignore differentiation of Spark patch levels
-sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", (if (scalaBinaryVersion.value == "2.10") "1.6.3" else "2.1.1"))
+sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "2.1.1")
sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
----------------------------------------------------------------------
diff --git a/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala b/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
deleted file mode 100644
index 0652e0b..0000000
--- a/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data
-
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
-
-object SparkVersionDependent {
-
- def sqlSession(sc: SparkContext): SQLContext = {
- return new SQLContext(sc)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/docs/manual/source/install/index.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/index.html.md.erb b/docs/manual/source/install/index.html.md.erb
index e666b69..c8bd44a 100644
--- a/docs/manual/source/install/index.html.md.erb
+++ b/docs/manual/source/install/index.html.md.erb
@@ -25,7 +25,7 @@ It is **very important** to meet the minimum version of the following
technologies that power Apache PredictionIO®.
* Apache Hadoop 2.6.5 (optional, required only if YARN and HDFS are needed)
-* Apache Spark 1.6.3 for Hadoop 2.6
+* Apache Spark 2.0.2 for Hadoop 2.6
* Java SE Development Kit 8
and one of the following sets:
@@ -41,8 +41,8 @@ or
* Apache HBase 0.98.5
* Elasticsearch 1.7.6
-WARNING: **Note that support for Scala 2.10 and Spark 1.6.x are deprecated as of PredictionIO 0.13.0,
-and may be removed in a future release.**
+WARNING: **Note that support for Scala 2.10 and Spark 1.6.x has been dropped in PredictionIO 0.14.0.
+They are no longer supported.**
If you are running on a single machine, we recommend a minimum of 2GB memory.
http://git-wip-us.apache.org/repos/asf/predictionio/blob/718ebe2e/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index cd71fdd..8c4c6ae 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -109,7 +109,7 @@ object Management extends EitherLogging {
val sparkHomePath = Common.getSparkHome(sparkHome)
if (new File(s"$sparkHomePath/bin/spark-submit").exists) {
info(s"Apache Spark is installed at $sparkHomePath")
- val sparkMinVersion = "1.6.3"
+ val sparkMinVersion = "2.0.2"
pioStatus = pioStatus.copy(
sparkHome = sparkHomePath,
sparkMinVersion = sparkMinVersion)