You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2016/07/26 18:57:24 UTC
[1/3] incubator-predictionio git commit: Replace the deprecated usage
of Spark API
Repository: incubator-predictionio
Updated Branches:
refs/heads/develop c6be80071 -> 3e4605f4a
Replace the deprecated usage of Spark API
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/da70a5ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/da70a5ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/da70a5ab
Branch: refs/heads/develop
Commit: da70a5abd1f845d5efbb991d33b629f14d054122
Parents: 034f5a0
Author: hyukjinkwon <gu...@gmail.com>
Authored: Sun Jul 24 11:48:10 2016 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Sun Jul 24 11:48:10 2016 +0900
----------------------------------------------------------------------
.../scala/org/apache/predictionio/data/view/DataView.scala | 7 ++++---
.../org/apache/predictionio/tools/export/EventsToFile.scala | 5 +++--
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/da70a5ab/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala b/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
index 31937d5..61ad11a 100644
--- a/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
@@ -23,6 +23,7 @@ import org.apache.predictionio.data.store.PEventStore
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.SQLContext
import org.joda.time.DateTime
@@ -83,7 +84,7 @@ object DataView {
val baseDir = s"${sys.env("PIO_FS_BASEDIR")}/view"
val fileName = s"$baseDir/$name-$appName-$hash.parquet"
try {
- sqlContext.parquetFile(fileName)
+ sqlContext.read.parquet(fileName)
} catch {
case e: java.io.FileNotFoundException =>
logger.info("Cached copy not found, reading from DB.")
@@ -97,8 +98,8 @@ object DataView {
import sqlContext.implicits._ // needed for RDD.toDF()
val resultDF = result.toDF()
- resultDF.saveAsParquetFile(fileName)
- sqlContext.parquetFile(fileName)
+ resultDF.write.mode(SaveMode.ErrorIfExists).parquet(fileName)
+ sqlContext.read.parquet(fileName)
case e: java.lang.RuntimeException =>
if (e.toString.contains("is not a Parquet file")) {
logger.error(s"$fileName does not contain a valid Parquet file. " +
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/da70a5ab/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
index feabce4..bf732b7 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
@@ -23,6 +23,7 @@ import org.apache.predictionio.workflow.WorkflowContext
import org.apache.predictionio.workflow.WorkflowUtils
import grizzled.slf4j.Logging
+import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.SQLContext
import org.json4s.native.Serialization._
@@ -94,8 +95,8 @@ object EventsToFile extends Logging {
if (args.format == "json") {
jsonStringRdd.saveAsTextFile(args.outputPath)
} else {
- val jsonRdd = sqlContext.jsonRDD(jsonStringRdd)
- jsonRdd.saveAsParquetFile(args.outputPath)
+ val jsonDf = sqlContext.read.json(jsonStringRdd)
+ jsonDf.write.mode(SaveMode.ErrorIfExists).parquet(args.outputPath)
}
info(s"Events are exported to ${args.outputPath}/.")
info("Done.")
[2/3] incubator-predictionio git commit: Update minimal requirement
for Spark from 1.3.0 to 1.4.0
Posted by do...@apache.org.
Update minimal requirement for Spark from 1.3.0 to 1.4.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e1374414
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e1374414
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e1374414
Branch: refs/heads/develop
Commit: e1374414c3da03a894c6b44d988f52a28429578a
Parents: da70a5a
Author: hyukjinkwon <gu...@gmail.com>
Authored: Tue Jul 26 09:31:15 2016 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Tue Jul 26 09:31:15 2016 +0900
----------------------------------------------------------------------
docs/manual/source/install/index.html.md.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e1374414/docs/manual/source/install/index.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/index.html.md.erb b/docs/manual/source/install/index.html.md.erb
index b5c5854..a4c3edc 100644
--- a/docs/manual/source/install/index.html.md.erb
+++ b/docs/manual/source/install/index.html.md.erb
@@ -27,7 +27,7 @@ It is **very important** to meet the minimum version of the following
technologies that power PredictionIO.
* Apache Hadoop 2.4.0 (optional, required only if YARN and HDFS are needed)
-* Apache Spark 1.3.0 for Hadoop 2.4
+* Apache Spark 1.4.0 for Hadoop 2.4
* Java SE Development Kit 7
and one of the following sets:
[3/3] incubator-predictionio git commit: Merge remote-tracking branch
'hyukjinkwon/deprecated-api-use' into develop
Posted by do...@apache.org.
Merge remote-tracking branch 'hyukjinkwon/deprecated-api-use' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/3e4605f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/3e4605f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/3e4605f4
Branch: refs/heads/develop
Commit: 3e4605f4a9c664e4f477978e89d3201d2ffe500e
Parents: c6be800 e137441
Author: Donald Szeto <do...@apache.org>
Authored: Tue Jul 26 11:57:05 2016 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Jul 26 11:57:05 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/data/view/DataView.scala | 7 ++++---
docs/manual/source/install/index.html.md.erb | 2 +-
.../org/apache/predictionio/tools/export/EventsToFile.scala | 5 +++--
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------