You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@predictionio.apache.org by do...@apache.org on 2017/04/25 22:11:24 UTC
[01/50] [abbrv] incubator-predictionio git commit: [PIO-53]
Per-commit unit and integration tests
Repository: incubator-predictionio
Updated Branches:
refs/heads/livedoc ef0e26941 -> e34a853d0
[PIO-53] Per-commit unit and integration tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/f32b9415
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/f32b9415
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/f32b9415
Branch: refs/heads/livedoc
Commit: f32b941563eeba44d5d8b8299e404e98db69dec0
Parents: bab594a
Author: Donald Szeto <do...@apache.org>
Authored: Sun Feb 26 15:09:36 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Feb 26 15:09:36 2017 -0800
----------------------------------------------------------------------
.travis.yml | 5 +++
.../predictionio/workflow/WorkflowUtils.scala | 46 +++++++++++---------
tests/Dockerfile | 4 +-
tests/docker-files/init.sh | 3 +-
tests/docker-files/wait-for-postgres.sh | 5 ---
.../org/apache/predictionio/tools/Runner.scala | 14 +++---
6 files changed, 42 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 22138fc..b763e2d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -51,8 +51,13 @@ env:
before_install:
- unset SBT_OPTS JVM_OPTS
+ - sudo rm /usr/local/bin/docker-compose
+ - curl -L https://github.com/docker/compose/releases/download/1.11.1/docker-compose-`uname -s`-`uname -m` > docker-compose
+ - chmod +x docker-compose
+ - sudo mv docker-compose /usr/local/bin
before_script:
+ - docker-compose -v
- ./tests/before_script.travis.sh
script:
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
index 93a676a..0e578be 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/WorkflowUtils.scala
@@ -19,28 +19,17 @@
package org.apache.predictionio.workflow
import java.io.File
-import java.io.FileNotFoundException
-
-import org.apache.predictionio.controller.EmptyParams
-import org.apache.predictionio.controller.EngineFactory
-import org.apache.predictionio.controller.EngineParamsGenerator
-import org.apache.predictionio.controller.Evaluation
-import org.apache.predictionio.controller.Params
-import org.apache.predictionio.controller.PersistentModelLoader
-import org.apache.predictionio.controller.Utils
-import org.apache.predictionio.core.BuildInfo
-
-import com.google.gson.Gson
-import com.google.gson.JsonSyntaxException
+import java.net.URI
+
+import com.google.gson.{Gson, JsonSyntaxException}
import grizzled.slf4j.Logging
+import org.apache.log4j.{Level, LogManager}
+import org.apache.predictionio.controller._
import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
-import org.apache.log4j.Level
-import org.apache.log4j.LogManager
import org.apache.spark.SparkContext
import org.apache.spark.api.java.JavaRDDLike
import org.apache.spark.rdd.RDD
import org.json4s.JsonAST.JValue
-import org.json4s.MappingException
import org.json4s._
import org.json4s.native.JsonMethods._
@@ -244,12 +233,12 @@ object WorkflowUtils extends Logging {
"HADOOP_CONF_DIR" -> "core-site.xml",
"HBASE_CONF_DIR" -> "hbase-site.xml")
- thirdPartyFiles.keys.toSeq.map { k: String =>
+ thirdPartyFiles.keys.toSeq.flatMap { k: String =>
sys.env.get(k) map { x =>
val p = Seq(x, thirdPartyFiles(k)).mkString(File.separator)
if (new File(p).exists) Seq(p) else Seq[String]()
} getOrElse Seq[String]()
- }.flatten
+ }
}
def thirdPartyClasspaths: Seq[String] = {
@@ -260,9 +249,26 @@ object WorkflowUtils extends Logging {
"MYSQL_JDBC_DRIVER",
"HADOOP_CONF_DIR",
"HBASE_CONF_DIR")
- thirdPartyPaths.map(p =>
+ thirdPartyPaths.flatMap(p =>
sys.env.get(p).map(Seq(_)).getOrElse(Seq[String]())
- ).flatten
+ )
+ }
+
+ def thirdPartyJars: Seq[URI] = {
+ val thirdPartyPaths = Seq(
+ "POSTGRES_JDBC_DRIVER",
+ "MYSQL_JDBC_DRIVER")
+ thirdPartyPaths.flatMap(p =>
+ sys.env.get(p) map { f =>
+ val file = new File(f)
+ if (file.exists()) {
+ Seq(file.toURI)
+ } else {
+ warn(s"Environment variable $p is pointing to a nonexistent file $f. Ignoring.")
+ Seq.empty[URI]
+ }
+ } getOrElse Seq.empty[URI]
+ )
}
def modifyLogging(verbose: Boolean): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index 9f4decd..dff4fa4 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -24,7 +24,7 @@ ENV HBASE_VERSION 1.0.0
ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
-COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers
+COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
COPY docker-files/init.sh init.sh
COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh
@@ -46,5 +46,7 @@ EXPOSE 8000
# eventserver
EXPOSE 7070
+ENV SLEEP_TIME 30
+
ENTRYPOINT ["/init.sh"]
CMD 'bash'
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/tests/docker-files/init.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/init.sh b/tests/docker-files/init.sh
index af68f1b..8dc08ea 100755
--- a/tests/docker-files/init.sh
+++ b/tests/docker-files/init.sh
@@ -17,6 +17,7 @@
#
set -e
-/wait-for-postgres.sh postgres /bin/true
export PYTHONPATH=/$PIO_HOME/tests:$PYTHONPATH
+echo "Sleeping $SLEEP_TIME seconds for all services to be ready..."
+sleep $SLEEP_TIME
eval $@
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/tests/docker-files/wait-for-postgres.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/wait-for-postgres.sh b/tests/docker-files/wait-for-postgres.sh
index 2c7d855..967e5fc 100755
--- a/tests/docker-files/wait-for-postgres.sh
+++ b/tests/docker-files/wait-for-postgres.sh
@@ -18,13 +18,8 @@
set -e
host="$1"
-shift
-cmd="$@"
until psql -h "$host" -U "pio" -c '\l'; do
>&2 echo "Postgres is unavailable - sleeping"
sleep 1
done
-
->&2 echo "Postgres is up - executing command"
-exec $cmd
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f32b9415/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index d9752df..f98dda1 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -15,18 +15,15 @@
* limitations under the License.
*/
-
package org.apache.predictionio.tools
import java.io.File
import java.net.URI
-import org.apache.predictionio.tools.console.ConsoleArgs
-import org.apache.predictionio.workflow.WorkflowUtils
-import org.apache.predictionio.tools.ReturnTypes._
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.predictionio.tools.ReturnTypes._
+import org.apache.predictionio.workflow.WorkflowUtils
import scala.sys.process._
@@ -163,8 +160,9 @@ object Runner extends EitherLogging {
val sparkSubmitCommand =
Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator))
- val sparkSubmitJars = if (extraJars.nonEmpty) {
- Seq("--jars", deployedJars.map(_.toString).mkString(","))
+ val sparkSubmitJarsList = WorkflowUtils.thirdPartyJars ++ deployedJars
+ val sparkSubmitJars = if (sparkSubmitJarsList.nonEmpty) {
+ Seq("--jars", sparkSubmitJarsList.map(_.toString).mkString(","))
} else {
Nil
}
[16/50] [abbrv] incubator-predictionio git commit: [PIO-49]
Additional improvement to ES5 support
Posted by do...@apache.org.
[PIO-49] Additional improvement to ES5 support
* Add option to build ES1 or ES5-default distribution
* Aggregate all storage subprojects
* Update deprecated SBT code
* Update Docker Compose to use official Elastic images
* Update Travis to use new build option
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/31c4bd19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/31c4bd19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/31c4bd19
Branch: refs/heads/livedoc
Commit: 31c4bd1927eb9ae753f82b39b71ff3df7ca779de
Parents: d78b3cb
Author: Donald Szeto <do...@apache.org>
Authored: Tue Mar 7 23:16:33 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 23:16:33 2017 -0800
----------------------------------------------------------------------
.travis.yml | 4 +--
build.sbt | 6 +++++
make-distribution.sh | 47 ++++++++++++++++++++++++++++++++++-
storage/elasticsearch/build.sbt | 3 +--
storage/elasticsearch1/build.sbt | 2 +-
storage/hbase/build.sbt | 2 +-
storage/hdfs/build.sbt | 2 +-
storage/jdbc/build.sbt | 2 +-
storage/localfs/build.sbt | 2 +-
tests/build-docker.sh | 8 ++++--
tests/docker-compose.yml | 4 ++-
11 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b763e2d..799ec3d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,8 +46,8 @@ env:
matrix:
- BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=5
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=5
before_install:
- unset SBT_OPTS JVM_OPTS
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 98444b9..fa5ba29 100644
--- a/build.sbt
+++ b/build.sbt
@@ -166,6 +166,12 @@ val root = (project in file(".")).
common,
core,
data,
+ dataElasticsearch1,
+ dataElasticsearch,
+ dataHbase,
+ dataHdfs,
+ dataJdbc,
+ dataLocalfs,
tools,
e2)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 31954c0..c360c0e 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -19,6 +19,45 @@
set -e
+usage ()
+{
+ echo "Usage: $0 [-h|--help] [--with-es=x]"
+ echo ""
+ echo " -h|--help Show usage"
+ echo ""
+ echo " --with-es=1 Build distribution with Elasticsearch 1 support as default"
+ echo " --with-es=5 Build distribution with Elasticsearch 5 support as default"
+}
+
+ES_VERSION=1
+
+for i in "$@"
+do
+case $i in
+ -h|--help)
+ usage
+ shift
+ exit
+ ;;
+ --with-es=*)
+ ES_VERSION="${i#*=}"
+ shift
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+esac
+done
+
+if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ]
+then
+ echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m"
+else
+ usage
+ exit 1
+fi
+
FWDIR="$(cd `dirname $0`; pwd)"
DISTDIR="${FWDIR}/dist"
@@ -45,7 +84,13 @@ cp ${FWDIR}/project/build.properties ${DISTDIR}/project
cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
-cp ${FWDIR}/assembly/extra/*jar ${DISTDIR}/lib/extra
+
+if [ "$ES_VERSION" = "5" ]
+then
+ mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra
+else
+ mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra
+fi
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index b22cbd8..925b66f 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -53,5 +53,4 @@ assemblyShadeRules in assembly := Seq(
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
-
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index 8c29b84..457aba2 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "extra" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 5856a5e..673697b 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -52,5 +52,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 9f064c6..35c9153 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 63d420b..0a07ee0 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -43,5 +43,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index 2cf9977..f94b355 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -40,5 +40,5 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
+assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index 459b929..a899a18 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/usr/bin/env bash -ex
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -29,7 +29,11 @@ fi
docker pull predictionio/pio-testing-base
pushd $DIR/..
-./make-distribution.sh
+if [ -z "$ES_VERSION" ]; then
+ ./make-distribution.sh
+else
+ ./make-distribution.sh --with-es=$ES_VERSION
+fi
sbt/sbt clean
mkdir assembly
cp dist/lib/*.jar assembly/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/31c4bd19/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index b556f7b..b6774d3 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,9 @@
version: "2"
services:
elasticsearch:
- image: elasticsearch:5-alpine
+ image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+ environment:
+ - xpack.security.enabled=false
hbase:
image: harisekhon/hbase:1.0
postgres:
[35/50] [abbrv] incubator-predictionio git commit: Rebuild on Travis
Posted by do...@apache.org.
Rebuild on Travis
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/5f7a59ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/5f7a59ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/5f7a59ad
Branch: refs/heads/livedoc
Commit: 5f7a59ad413a04021ceee74063b8f045b3273d31
Parents: 14b4998
Author: Donald Szeto <do...@apache.org>
Authored: Mon Mar 20 22:03:54 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 20 22:03:54 2017 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[39/50] [abbrv] incubator-predictionio git commit: Update release
notes
Posted by do...@apache.org.
Update release notes
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/dfb01e32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/dfb01e32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/dfb01e32
Branch: refs/heads/livedoc
Commit: dfb01e327f84efbb4823d5c2f962516b4bae6d31
Parents: 4dcb271
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 30 11:27:17 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 30 11:27:17 2017 -0700
----------------------------------------------------------------------
RELEASE.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/dfb01e32/RELEASE.md
----------------------------------------------------------------------
diff --git a/RELEASE.md b/RELEASE.md
index baa5246..3cc9259 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -61,8 +61,8 @@ Apr ?, 2017
The following contributors have spent a great deal of effort to bring to you
this release:
-Ahmet DAL, Alexander Merritt, Bansari Shah, Chan Lee, Chris Woodford, Daniel
-Gabrieli, Dennis Jung, Donald Szeto, Emily Rose, Hari Charan Ayada,
+Ahmet DAL, Alexander Merritt, Amy Lin, Bansari Shah, Chan Lee, Chris Woodford,
+Daniel Gabrieli, Dennis Jung, Donald Szeto, Emily Rose, Hari Charan Ayada,
infoquestsolutions, Jonny Daenen, Kenneth Chan, Laertis Pappas, Marcin
Ziemi\u0144ski, Naoki Takezoe, Rajdeep Dua, Shinsuke Sugaya, Pat Ferrel, scorpiovn,
Suneel Marthi, Steven Yan, Takahiro Hagino, Takako Shimamoto
[09/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by do...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/8fd59fdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/8fd59fdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/8fd59fdf
Branch: refs/heads/livedoc
Commit: 8fd59fdf1ad194e500c48dc5e333b488e7d05a7a
Parents: 19c47ad 3525049
Author: Donald Szeto <do...@apache.org>
Authored: Tue Mar 7 10:54:53 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 10:54:53 2017 -0800
----------------------------------------------------------------------
docs/manual/data/nav/main.yml | 89 ++++++++++++++++++++
.../datacollection/eventmodel.html.md.erb | 2 +-
docs/manual/source/gallery/templates.yaml | 14 +--
docs/manual/source/partials/_footer.html.slim | 4 +-
docs/manual/source/templates/index.html.md | 11 +++
.../templates/recommendation/how-to.html.md | 1 +
6 files changed, 111 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8fd59fdf/docs/manual/source/datacollection/eventmodel.html.md.erb
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8fd59fdf/docs/manual/source/gallery/templates.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/8fd59fdf/docs/manual/source/templates/recommendation/how-to.html.md
----------------------------------------------------------------------
[44/50] [abbrv] incubator-predictionio git commit: ES 2.x has been
fixed by #352
Posted by do...@apache.org.
ES 2.x has been fixed by #352
Closes #195
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/caa43f86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/caa43f86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/caa43f86
Branch: refs/heads/livedoc
Commit: caa43f866d421eb03f22b2c0123734c4db37efd1
Parents: ba46e2c
Author: Donald Szeto <do...@apache.org>
Authored: Mon Apr 3 10:52:10 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Apr 3 10:52:10 2017 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[30/50] [abbrv] incubator-predictionio git commit: [PIO-30] Set up a
cross build for Spark 2.0 and Scala 2.11
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-files/env-conf/pio-env.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh
index 0acf3a7..16f1fef 100644
--- a/tests/docker-files/env-conf/pio-env.sh
+++ b/tests/docker-files/env-conf/pio-env.sh
@@ -24,9 +24,9 @@
# you need to change these to fit your site.
# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$SPARK_HOME
+# SPARK_HOME=$SPARK_HOME
-POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
+POSTGRES_JDBC_DRIVER=/drivers/$PGSQL_JAR
MYSQL_JDBC_DRIVER=
# ES_CONF_DIR: You must configure this if you have advanced configuration for
@@ -88,7 +88,16 @@ PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
-PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
+if [ ! -z "$PIO_ELASTICSEARCH_VERSION" ]; then
+ ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
+else
+ ES_MAJOR=1
+fi
+if [ "$ES_MAJOR" = "1" ]; then
+ PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
+else
+ PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
+fi
#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
# Local File System Example
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-files/init.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/init.sh b/tests/docker-files/init.sh
index 8dc08ea..fc12ffe 100755
--- a/tests/docker-files/init.sh
+++ b/tests/docker-files/init.sh
@@ -17,7 +17,7 @@
#
set -e
-export PYTHONPATH=/$PIO_HOME/tests:$PYTHONPATH
+export PYTHONPATH=$PIO_HOME/tests:$PYTHONPATH
echo "Sleeping $SLEEP_TIME seconds for all services to be ready..."
sleep $SLEEP_TIME
eval $@
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-files/set_build_profile.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/set_build_profile.sh b/tests/docker-files/set_build_profile.sh
new file mode 100755
index 0000000..141dd46
--- /dev/null
+++ b/tests/docker-files/set_build_profile.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Sets version of profile dependencies from sbt configuration.
+# eg. Run `source ./set_build_profile.sh scala-2.11`
+
+set -e
+
+if [[ "$#" -ne 1 ]]; then
+ echo "Usage: set-build-profile.sh <build-profile>"
+ exit 1
+fi
+
+set -a
+eval `$PIO_HOME/sbt/sbt --error 'set showSuccess := false' -Dbuild.profile=$1 printProfile | grep '.*_VERSION=.*'`
+set +a
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/pio_tests/engines/recommendation-engine/build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt
index 52e8742..13fe354 100644
--- a/tests/pio_tests/engines/recommendation-engine/build.sbt
+++ b/tests/pio_tests/engines/recommendation-engine/build.sbt
@@ -19,11 +19,13 @@ import AssemblyKeys._
assemblySettings
+scalaVersion in ThisBuild := sys.env.getOrElse("PIO_SCALA_VERSION", "2.10.5")
+
name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % pioVersion.value % "provided",
- "org.apache.spark" %% "spark-core" % "1.3.0" % "provided",
- "org.apache.spark" %% "spark-mllib" % "1.3.0" % "provided")
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-SNAPSHOT" % "provided",
+ "org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided",
+ "org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/pio_tests/engines/recommendation-engine/manifest.json
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/manifest.json b/tests/pio_tests/engines/recommendation-engine/manifest.json
deleted file mode 100644
index 9c545ce..0000000
--- a/tests/pio_tests/engines/recommendation-engine/manifest.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"yDBpzjz39AjIxlOAh8W4t3QSc75uPCuJ","version":"ee98ff9c009ef0d9fb1bc6b78750b83a0ceb37b2","name":"recommendation-engine","description":"pio-autogen-manifest","files":[],"engineFactory":""}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt b/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
deleted file mode 100644
index 8346a96..0000000
--- a/tests/pio_tests/engines/recommendation-engine/project/pio-build.sbt
+++ /dev/null
@@ -1 +0,0 @@
-addSbtPlugin("io.prediction" % "pio-build" % "0.9.0")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/pio_tests/scenarios/quickstart_test.py
----------------------------------------------------------------------
diff --git a/tests/pio_tests/scenarios/quickstart_test.py b/tests/pio_tests/scenarios/quickstart_test.py
index 1c5f422..ab7180d 100644
--- a/tests/pio_tests/scenarios/quickstart_test.py
+++ b/tests/pio_tests/scenarios/quickstart_test.py
@@ -89,8 +89,8 @@ class QuickStartTest(BaseTestCase):
self.app.build(engine_dir=engine_path)
self.log.info("Training...")
self.app.train(engine_dir=engine_path)
- self.log.info("Deploying and waiting 15s for it to start...")
- self.app.deploy(wait_time=15, engine_dir=engine_path)
+ self.log.info("Deploying and waiting 30s for it to start...")
+ self.app.deploy(wait_time=30, engine_dir=engine_path)
self.log.info("Sending a single query and checking results")
user_query = { "user": 1, "num": 4 }
@@ -153,8 +153,8 @@ class QuickStartTest(BaseTestCase):
self.app.build()
self.log.info("Training...")
self.app.train()
- self.log.info("Deploying and waiting 15s for it to start...")
- self.app.deploy(wait_time=15)
+ self.log.info("Deploying and waiting 35s for it to start...")
+ self.app.deploy(wait_time=35)
self.log.info("Testing pio commands outside of engine directory")
self.engine_dir_test()
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/pio_tests/utils.py
----------------------------------------------------------------------
diff --git a/tests/pio_tests/utils.py b/tests/pio_tests/utils.py
index e6c5b0b..05c8d1c 100644
--- a/tests/pio_tests/utils.py
+++ b/tests/pio_tests/utils.py
@@ -151,10 +151,11 @@ def import_events_batch(events, test_context, appid, channel=None):
try:
with open(file_path, 'w') as f:
f.write(contents)
- srun('pio import --appid {} --input {} {}'.format(
+ srun('pio import --appid {} --input {} {} -- {}'.format(
appid,
file_path,
- '--channel {}'.format(channel) if channel else ''))
+ '--channel {}'.format(channel) if channel else '',
+ '--conf spark.sql.warehouse.dir=file:///tmp/spark-warehouse'))
finally:
os.remove(file_path)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index 6257fa2..ad7e189 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -46,17 +46,10 @@ shift
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ "$ES_VERSION" = "1" ]; then
- docker-compose -f $DIR/docker-compose-es1.yml run \
- -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
- -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
- -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
- pio-testing $@
-else
- docker-compose -f $DIR/docker-compose.yml run \
- -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
- -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
- -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
- pio-testing $@
-fi
+source $DIR/../conf/pio-vendors.sh
+docker-compose -f $DIR/docker-compose.yml run \
+ -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+ -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+ -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+ pio-testing $@
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/script.travis.sh b/tests/script.travis.sh
index db69413..1d4985e 100755
--- a/tests/script.travis.sh
+++ b/tests/script.travis.sh
@@ -17,9 +17,9 @@
#
if [[ $BUILD_TYPE == Unit ]]; then
- ./tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
- /PredictionIO/tests/unit.sh
+ tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
+ "/PredictionIO/tests/unit.sh"
else
- ./tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
- python3 /PredictionIO/tests/pio_tests/tests.py
+ tests/run_docker.sh $METADATA_REP $EVENTDATA_REP $MODELDATA_REP \
+ "python3 /PredictionIO/tests/pio_tests/tests.py"
fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/unit.sh
----------------------------------------------------------------------
diff --git a/tests/unit.sh b/tests/unit.sh
index 6382a70..1421dce 100755
--- a/tests/unit.sh
+++ b/tests/unit.sh
@@ -14,20 +14,31 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
-# Run license check
pushd /PredictionIO
+# Run license check
./tests/check_license.sh
# Prepare pio environment variables
set -a
-source conf/pio-env.sh
+source ./conf/pio-env.sh
set +a
+source ./conf/pio-vendors.sh
# Run stylecheck
-sbt/sbt scalastyle
+sbt/sbt scalastyle \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
+
# Run all unit tests
-sbt/sbt test
+sbt/sbt dataJdbc/compile test storage/test \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
popd
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tools/build.sbt
----------------------------------------------------------------------
diff --git a/tools/build.sbt b/tools/build.sbt
index 57e7d96..483a591 100644
--- a/tools/build.sbt
+++ b/tools/build.sbt
@@ -15,27 +15,17 @@
* limitations under the License.
*/
+import PIOBuild._
import sbtassembly.AssemblyPlugin.autoImport._
name := "apache-predictionio-tools"
libraryDependencies ++= Seq(
- "com.github.scopt" %% "scopt" % "3.2.0",
- "io.spray" %% "spray-can" % "1.3.3",
- "io.spray" %% "spray-routing" % "1.3.3",
"me.lessis" % "semverfi_2.10" % "0.1.3",
- "org.apache.hadoop" % "hadoop-common" % "2.6.2",
- "org.apache.hadoop" % "hadoop-hdfs" % "2.6.2",
- "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
- "org.clapper" %% "grizzled-slf4j" % "1.0.2",
- "org.json4s" %% "json4s-native" % json4sVersion.value,
- "org.json4s" %% "json4s-ext" % json4sVersion.value,
- "org.scalaj" %% "scalaj-http" % "1.1.6",
- "org.spark-project.akka" %% "akka-actor" % "2.3.4-spark",
+ "com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value,
"io.spray" %% "spray-testkit" % "1.3.3" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test",
- "org.spark-project.akka" %% "akka-slf4j" % "2.3.4-spark")
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.18"
@@ -49,12 +39,8 @@ assemblyMergeStrategy in assembly := {
excludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
cp filter { _.data.getName match {
- case "asm-3.1.jar" => true
- case "commons-beanutils-1.7.0.jar" => true
case "reflectasm-1.10.1.jar" => true
- case "commons-beanutils-core-1.8.0.jar" => true
case "kryo-3.0.3.jar" => true
- case "slf4j-log4j12-1.7.5.jar" => true
case _ => false
}}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
index bbe39a5..7e8fd30 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/admin/AdminAPI.scala
@@ -34,6 +34,7 @@ import spray.httpx.Json4sSupport
import spray.routing._
import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.Duration
class AdminServiceActor(val commandClient: CommandClient)
extends HttpServiceActor {
@@ -151,7 +152,7 @@ object AdminServer {
}
object AdminRun {
- def main (args: Array[String]) {
+ def main (args: Array[String]) : Unit = {
AdminServer.createAdminServer(AdminServerConfig(
ip = "localhost",
port = 7071))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
index de09cab..c101d3f 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/export/EventsToFile.scala
@@ -21,13 +21,13 @@ package org.apache.predictionio.tools.export
import org.apache.predictionio.controller.Utils
import org.apache.predictionio.data.storage.EventJson4sSupport
import org.apache.predictionio.data.storage.Storage
+import org.apache.predictionio.data.SparkVersionDependent
import org.apache.predictionio.tools.Runner
import org.apache.predictionio.workflow.WorkflowContext
import org.apache.predictionio.workflow.WorkflowUtils
import grizzled.slf4j.Logging
import org.apache.spark.sql.SaveMode
-import org.apache.spark.sql.SQLContext
import org.json4s.native.Serialization._
case class EventsToFileArgs(
@@ -91,14 +91,14 @@ object EventsToFile extends Logging {
mode = "Export",
batch = "App ID " + args.appId + channelStr,
executorEnv = Runner.envStringToMap(args.env))
- val sqlContext = new SQLContext(sc)
+ val sqlSession = SparkVersionDependent.sqlSession(sc)
val events = Storage.getPEvents()
val eventsRdd = events.find(appId = args.appId, channelId = channelId)(sc)
val jsonStringRdd = eventsRdd.map(write(_))
if (args.format == "json") {
jsonStringRdd.saveAsTextFile(args.outputPath)
} else {
- val jsonDf = sqlContext.read.json(jsonStringRdd)
+ val jsonDf = sqlSession.read.json(jsonStringRdd)
jsonDf.write.mode(SaveMode.ErrorIfExists).parquet(args.outputPath)
}
info(s"Events are exported to ${args.outputPath}/.")
[38/50] [abbrv] incubator-predictionio git commit: Merge branch
'livedoc' into develop
Posted by do...@apache.org.
Merge branch 'livedoc' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/4dcb2713
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/4dcb2713
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/4dcb2713
Branch: refs/heads/livedoc
Commit: 4dcb2713dc6a9cc5504b7562bb0b76c174317bfe
Parents: bea82a6 ef0e269
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 30 11:26:49 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 30 11:26:49 2017 -0700
----------------------------------------------------------------------
docs/manual/source/resources/faq.html.md | 37 +++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4dcb2713/docs/manual/source/resources/faq.html.md
----------------------------------------------------------------------
[13/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala b/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
deleted file mode 100644
index b453820..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.view
-
-import org.apache.predictionio.data.storage.hbase.HBPEvents
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventValidation
-import org.apache.predictionio.data.storage.DataMap
-import org.apache.predictionio.data.storage.Storage
-
-import org.joda.time.DateTime
-
-import org.json4s.JValue
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-
-// each JValue data associated with the time it is set
-private[predictionio] case class PropTime(val d: JValue, val t: Long) extends Serializable
-
-private[predictionio] case class SetProp (
- val fields: Map[String, PropTime],
- // last set time. Note: fields could be empty with valid set time
- val t: Long) extends Serializable {
-
- def ++ (that: SetProp): SetProp = {
- val commonKeys = fields.keySet.intersect(that.fields.keySet)
-
- val common: Map[String, PropTime] = commonKeys.map { k =>
- val thisData = this.fields(k)
- val thatData = that.fields(k)
- // only keep the value with latest time
- val v = if (thisData.t > thatData.t) thisData else thatData
- (k, v)
- }.toMap
-
- val combinedFields = common ++
- (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
-
- // keep the latest set time
- val combinedT = if (this.t > that.t) this.t else that.t
-
- SetProp(
- fields = combinedFields,
- t = combinedT
- )
- }
-}
-
-private[predictionio] case class UnsetProp (fields: Map[String, Long]) extends Serializable {
- def ++ (that: UnsetProp): UnsetProp = {
- val commonKeys = fields.keySet.intersect(that.fields.keySet)
-
- val common: Map[String, Long] = commonKeys.map { k =>
- val thisData = this.fields(k)
- val thatData = that.fields(k)
- // only keep the value with latest time
- val v = if (thisData > thatData) thisData else thatData
- (k, v)
- }.toMap
-
- val combinedFields = common ++
- (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
-
- UnsetProp(
- fields = combinedFields
- )
- }
-}
-
-private[predictionio] case class DeleteEntity (t: Long) extends Serializable {
- def ++ (that: DeleteEntity): DeleteEntity = {
- if (this.t > that.t) this else that
- }
-}
-
-private[predictionio] case class EventOp (
- val setProp: Option[SetProp] = None,
- val unsetProp: Option[UnsetProp] = None,
- val deleteEntity: Option[DeleteEntity] = None
-) extends Serializable {
-
- def ++ (that: EventOp): EventOp = {
- EventOp(
- setProp = (setProp ++ that.setProp).reduceOption(_ ++ _),
- unsetProp = (unsetProp ++ that.unsetProp).reduceOption(_ ++ _),
- deleteEntity = (deleteEntity ++ that.deleteEntity).reduceOption(_ ++ _)
- )
- }
-
- def toDataMap(): Option[DataMap] = {
- setProp.flatMap { set =>
-
- val unsetKeys: Set[String] = unsetProp.map( unset =>
- unset.fields.filter{ case (k, v) => (v >= set.fields(k).t) }.keySet
- ).getOrElse(Set())
-
- val combinedFields = deleteEntity.map { delete =>
- if (delete.t >= set.t) {
- None
- } else {
- val deleteKeys: Set[String] = set.fields
- .filter { case (k, PropTime(kv, t)) =>
- (delete.t >= t)
- }.keySet
- Some(set.fields -- unsetKeys -- deleteKeys)
- }
- }.getOrElse{
- Some(set.fields -- unsetKeys)
- }
-
- // Note: mapValues() doesn't return concrete Map and causes
- // NotSerializableException issue. Use map(identity) to work around this.
- // see https://issues.scala-lang.org/browse/SI-7005
- combinedFields.map(f => DataMap(f.mapValues(_.d).map(identity)))
- }
- }
-
-}
-
-private[predictionio] object EventOp {
- def apply(e: Event): EventOp = {
- val t = e.eventTime.getMillis
- e.event match {
- case "$set" => {
- val fields = e.properties.fields.mapValues(jv =>
- PropTime(jv, t)
- ).map(identity)
-
- EventOp(
- setProp = Some(SetProp(fields = fields, t = t))
- )
- }
- case "$unset" => {
- val fields = e.properties.fields.mapValues(jv => t).map(identity)
- EventOp(
- unsetProp = Some(UnsetProp(fields = fields))
- )
- }
- case "$delete" => {
- EventOp(
- deleteEntity = Some(DeleteEntity(t))
- )
- }
- case _ => {
- EventOp()
- }
- }
- }
-}
-
-@deprecated("Use PEvents or PEventStore instead.", "0.9.2")
-class PBatchView(
- val appId: Int,
- val startTime: Option[DateTime],
- val untilTime: Option[DateTime],
- val sc: SparkContext) {
-
- // NOTE: parallel Events DB interface
- @transient lazy val eventsDb = Storage.getPEvents()
-
- @transient lazy val _events: RDD[Event] =
- eventsDb.getByAppIdAndTimeAndEntity(
- appId = appId,
- startTime = startTime,
- untilTime = untilTime,
- entityType = None,
- entityId = None)(sc)
-
- // TODO: change to use EventSeq?
- @transient lazy val events: RDD[Event] = _events
-
- def aggregateProperties(
- entityType: String,
- startTimeOpt: Option[DateTime] = None,
- untilTimeOpt: Option[DateTime] = None
- ): RDD[(String, DataMap)] = {
-
- _events
- .filter( e => ((e.entityType == entityType) &&
- (EventValidation.isSpecialEvents(e.event))) )
- .map( e => (e.entityId, EventOp(e) ))
- .aggregateByKey[EventOp](EventOp())(
- // within same partition
- seqOp = { case (u, v) => u ++ v },
- // across partition
- combOp = { case (accu, u) => accu ++ u }
- )
- .mapValues(_.toDataMap)
- .filter{ case (k, v) => v.isDefined }
- .map{ case (k, v) => (k, v.get) }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index b6c8ed3..31954c0 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -27,13 +27,15 @@ VERSION=$(grep version ${FWDIR}/build.sbt | grep ThisBuild | grep -o '".*"' | se
echo "Building binary distribution for PredictionIO $VERSION..."
cd ${FWDIR}
-sbt/sbt common/publishLocal data/publishLocal core/publishLocal e2/publishLocal tools/assembly
+sbt/sbt common/publishLocal data/publishLocal core/publishLocal e2/publishLocal dataElasticsearch1/assembly dataElasticsearch/assembly dataHbase/assembly dataHdfs/assembly dataJdbc/assembly dataLocalfs/assembly tools/assembly
cd ${FWDIR}
rm -rf ${DISTDIR}
mkdir -p ${DISTDIR}/bin
mkdir -p ${DISTDIR}/conf
mkdir -p ${DISTDIR}/lib
+mkdir -p ${DISTDIR}/lib/spark
+mkdir -p ${DISTDIR}/lib/extra
mkdir -p ${DISTDIR}/project
mkdir -p ${DISTDIR}/sbt
@@ -42,6 +44,8 @@ cp ${FWDIR}/conf/* ${DISTDIR}/conf
cp ${FWDIR}/project/build.properties ${DISTDIR}/project
cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
+cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
+cp ${FWDIR}/assembly/extra/*jar ${DISTDIR}/lib/extra
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/.gitignore
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/.gitignore b/storage/elasticsearch/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/elasticsearch/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
new file mode 100644
index 0000000..b22cbd8
--- /dev/null
+++ b/storage/elasticsearch/build.sbt
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-elasticsearch"
+
+elasticsearchVersion := "5.2.1"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
+ "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
+ "org.elasticsearch.client" % "rest" % elasticsearchVersion.value,
+ "org.elasticsearch" %% "elasticsearch-spark-13" % elasticsearchVersion.value
+ exclude("org.apache.spark", "spark-sql_2.10")
+ exclude("org.apache.spark", "spark-streaming_2.10"),
+ "org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value,
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+assemblyShadeRules in assembly := Seq(
+ ShadeRule.rename("org.apache.http.**" -> "shadeio.data.http.@1").inAll
+)
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
new file mode 100644
index 0000000..cb6d330
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters.mapAsJavaMapConverter
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.AccessKey
+import org.apache.predictionio.data.storage.AccessKeys
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+
+/** Elasticsearch implementation of AccessKeys. */
+class ESAccessKeys(client: ESClient, config: StorageClientConfig, index: String)
+ extends AccessKeys with Logging {
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "accesskeys"
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("key" -> ("type" -> "keyword")) ~
+ ("events" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def insert(accessKey: AccessKey): Option[String] = {
+ val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
+ update(accessKey.copy(key = key))
+ Some(key)
+ }
+
+ def get(id: String): Option[AccessKey] = {
+ if (id.isEmpty) {
+ return None
+ }
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "found").extract[Boolean] match {
+ case true =>
+ Some((jsonResponse \ "_source").extract[AccessKey])
+ case _ =>
+ None
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => None
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ }
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getAll(): Seq[AccessKey] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("match_all" -> List.empty))
+ ESUtils.getAll[AccessKey](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getByAppid(appid: Int): Seq[AccessKey] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("appid" -> appid)))
+ ESUtils.getAll[AccessKey](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def update(accessKey: AccessKey): Unit = {
+ val id = accessKey.key
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write(accessKey), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" =>
+ case "updated" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def delete(id: String): Unit = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "DELETE",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
new file mode 100644
index 0000000..abea2b8
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters.mapAsJavaMapConverter
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.App
+import org.apache.predictionio.data.storage.Apps
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+
+/** Elasticsearch implementation of Items. */
+class ESApps(client: ESClient, config: StorageClientConfig, index: String)
+ extends Apps with Logging {
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "apps"
+ private val seq = new ESSequences(client, config, index)
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("id" -> ("type" -> "keyword")) ~
+ ("name" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def insert(app: App): Option[Int] = {
+ val id =
+ if (app.id == 0) {
+ var roll = seq.genNext(estype)
+ while (!get(roll).isEmpty) roll = seq.genNext(estype)
+ roll
+ } else app.id
+ update(app.copy(id = id))
+ Some(id)
+ }
+
+ def get(id: Int): Option[App] = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "found").extract[Boolean] match {
+ case true =>
+ Some((jsonResponse \ "_source").extract[App])
+ case _ =>
+ None
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => None
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ }
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getByName(name: String): Option[App] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("name" -> name)))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/_search",
+ Map.empty[String, String].asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "hits" \ "total").extract[Long] match {
+ case 0 => None
+ case _ =>
+ val results = (jsonResponse \ "hits" \ "hits").extract[Seq[JValue]]
+ val result = (results.head \ "_source").extract[App]
+ Some(result)
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/_search", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getAll(): Seq[App] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("match_all" -> List.empty))
+ ESUtils.getAll[App](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def update(app: App): Unit = {
+ val id = app.id.toString
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write(app), ContentType.APPLICATION_JSON);
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" =>
+ case "updated" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def delete(id: Int): Unit = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "DELETE",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
new file mode 100644
index 0000000..f092cc7
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters.mapAsJavaMapConverter
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.Channel
+import org.apache.predictionio.data.storage.Channels
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+
+class ESChannels(client: ESClient, config: StorageClientConfig, index: String)
+ extends Channels with Logging {
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "channels"
+ private val seq = new ESSequences(client, config, index)
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("name" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def insert(channel: Channel): Option[Int] = {
+ val id =
+ if (channel.id == 0) {
+ var roll = seq.genNext(estype)
+ while (!get(roll).isEmpty) roll = seq.genNext(estype)
+ roll
+ } else channel.id
+
+ if (update(channel.copy(id = id))) Some(id) else None
+ }
+
+ def get(id: Int): Option[Channel] = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "found").extract[Boolean] match {
+ case true =>
+ Some((jsonResponse \ "_source").extract[Channel])
+ case _ =>
+ None
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => None
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ }
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getByAppid(appid: Int): Seq[Channel] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("appid" -> appid)))
+ ESUtils.getAll[Channel](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def update(channel: Channel): Boolean = {
+ val id = channel.id.toString
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write(channel), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "created" => true
+ case "updated" => true
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ false
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ false
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def delete(id: Int): Unit = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "DELETE",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
new file mode 100644
index 0000000..4dbacb7
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters.mapAsJavaMapConverter
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.EngineInstance
+import org.apache.predictionio.data.storage.EngineInstanceSerializer
+import org.apache.predictionio.data.storage.EngineInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+
+class ESEngineInstances(client: ESClient, config: StorageClientConfig, index: String)
+ extends EngineInstances with Logging {
+ implicit val formats = DefaultFormats + new EngineInstanceSerializer
+ private val estype = "engine_instances"
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("status" -> ("type" -> "keyword")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("engineId" -> ("type" -> "keyword")) ~
+ ("engineVersion" -> ("type" -> "keyword")) ~
+ ("engineVariant" -> ("type" -> "keyword")) ~
+ ("engineFactory" -> ("type" -> "keyword")) ~
+ ("batch" -> ("type" -> "keyword")) ~
+ ("dataSourceParams" -> ("type" -> "keyword")) ~
+ ("preparatorParams" -> ("type" -> "keyword")) ~
+ ("algorithmsParams" -> ("type" -> "keyword")) ~
+ ("servingParams" -> ("type" -> "keyword")) ~
+ ("status" -> ("type" -> "keyword"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def insert(i: EngineInstance): String = {
+ val id = i.id match {
+ case x if x.isEmpty =>
+ @scala.annotation.tailrec
+ def generateId(newId: Option[String]): String = {
+ newId match {
+ case Some(x) => x
+ case _ => generateId(preInsert())
+ }
+ }
+ generateId(preInsert())
+ case x => x
+ }
+
+ update(i.copy(id = id))
+ id
+ }
+
+ def preInsert(): Option[String] = {
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity("{}", ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" =>
+ Some((jsonResponse \ "_id").extract[String])
+ case _ =>
+ error(s"[$result] Failed to create $index/$estype")
+ None
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to create $index/$estype", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def get(id: String): Option[EngineInstance] = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "found").extract[Boolean] match {
+ case true =>
+ Some((jsonResponse \ "_source").extract[EngineInstance])
+ case _ =>
+ None
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => None
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ }
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getAll(): Seq[EngineInstance] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("match_all" -> List.empty))
+ ESUtils.getAll[EngineInstance](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Seq[EngineInstance] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("bool" ->
+ ("must" -> List(
+ ("term" ->
+ ("status" -> "COMPLETED")),
+ ("term" ->
+ ("engineId" -> engineId)),
+ ("term" ->
+ ("engineVersion" -> engineVersion)),
+ ("term" ->
+ ("engineVariant" -> engineVariant)))))) ~
+ ("sort" -> List(
+ ("startTime" ->
+ ("order" -> "desc"))))
+ ESUtils.getAll[EngineInstance](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getLatestCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Option[EngineInstance] =
+ getCompleted(
+ engineId,
+ engineVersion,
+ engineVariant).headOption
+
+ def update(i: EngineInstance): Unit = {
+ val id = i.id
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write(i), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" =>
+ case "updated" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def delete(id: String): Unit = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "DELETE",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
new file mode 100644
index 0000000..5bdc0fb
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters._
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.EvaluationInstance
+import org.apache.predictionio.data.storage.EvaluationInstanceSerializer
+import org.apache.predictionio.data.storage.EvaluationInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+
+class ESEvaluationInstances(client: ESClient, config: StorageClientConfig, index: String)
+ extends EvaluationInstances with Logging {
+ implicit val formats = DefaultFormats + new EvaluationInstanceSerializer
+ private val estype = "evaluation_instances"
+ private val seq = new ESSequences(client, config, index)
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("status" -> ("type" -> "keyword")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("evaluationClass" -> ("type" -> "keyword")) ~
+ ("engineParamsGeneratorClass" -> ("type" -> "keyword")) ~
+ ("batch" -> ("type" -> "keyword")) ~
+ ("evaluatorResults" -> ("type" -> "text") ~ ("index" -> "no")) ~
+ ("evaluatorResultsHTML" -> ("type" -> "text") ~ ("index" -> "no")) ~
+ ("evaluatorResultsJSON" -> ("type" -> "text") ~ ("index" -> "no"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def insert(i: EvaluationInstance): String = {
+ val id = i.id match {
+ case x if x.isEmpty =>
+ var roll = seq.genNext(estype).toString
+ while (!get(roll).isEmpty) roll = seq.genNext(estype).toString
+ roll
+ case x => x
+ }
+
+ update(i.copy(id = id))
+ id
+ }
+
+ def get(id: String): Option[EvaluationInstance] = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "found").extract[Boolean] match {
+ case true =>
+ Some((jsonResponse \ "_source").extract[EvaluationInstance])
+ case _ =>
+ None
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => None
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ }
+ case e: IOException =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getAll(): Seq[EvaluationInstance] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("match_all" -> List.empty))
+ ESUtils.getAll[EvaluationInstance](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def getCompleted(): Seq[EvaluationInstance] = {
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("status" -> "EVALCOMPLETED"))) ~
+ ("sort" ->
+ ("startTime" ->
+ ("order" -> "desc")))
+ ESUtils.getAll[EvaluationInstance](restClient, index, estype, compact(render(json)))
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ Nil
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def update(i: EvaluationInstance): Unit = {
+ val id = i.id
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write(i), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "created" =>
+ case "updated" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+
+ def delete(id: String): Unit = {
+ val restClient = client.open()
+ try {
+ val response = restClient.performRequest(
+ "DELETE",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava)
+ val json = parse(EntityUtils.toString(response.getEntity))
+ val result = (json \ "result").extract[String]
+ result match {
+ case "deleted" =>
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/$id", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
new file mode 100644
index 0000000..56f47ab
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import org.apache.hadoop.io.DoubleWritable
+import org.apache.hadoop.io.LongWritable
+import org.apache.hadoop.io.MapWritable
+import org.apache.hadoop.io.Text
+import org.apache.predictionio.data.storage.DataMap
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.EventValidation
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+import org.json4s._
+
+object ESEventsUtil {
+
+ implicit val formats = DefaultFormats
+
+ def resultToEvent(id: Text, result: MapWritable, appId: Int): Event = {
+
+ def getStringCol(col: String): String = {
+ val r = result.get(new Text(col)).asInstanceOf[Text]
+ require(r != null,
+ s"Failed to get value for column ${col}. " +
+ s"StringBinary: ${r.getBytes()}.")
+
+ r.toString()
+ }
+
+ def getOptStringCol(col: String): Option[String] = {
+ val r = result.get(new Text(col))
+ if (r == null) {
+ None
+ } else {
+ Some(r.asInstanceOf[Text].toString())
+ }
+ }
+
+ val tmp = result
+ .get(new Text("properties")).asInstanceOf[MapWritable]
+ .get(new Text("fields")).asInstanceOf[MapWritable]
+ .get(new Text("rating"))
+
+ val rating =
+ if (tmp.isInstanceOf[DoubleWritable]) tmp.asInstanceOf[DoubleWritable]
+ else if (tmp.isInstanceOf[LongWritable]) {
+ new DoubleWritable(tmp.asInstanceOf[LongWritable].get().toDouble)
+ }
+ else null
+
+ val properties: DataMap =
+ if (rating != null) DataMap(s"""{"rating":${rating.get().toString}}""")
+ else DataMap()
+
+
+ val eventId = Some(getStringCol("eventId"))
+ val event = getStringCol("event")
+ val entityType = getStringCol("entityType")
+ val entityId = getStringCol("entityId")
+ val targetEntityType = getOptStringCol("targetEntityType")
+ val targetEntityId = getOptStringCol("targetEntityId")
+ val prId = getOptStringCol("prId")
+ val eventTimeZone = getOptStringCol("eventTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+ val eventTime = new DateTime(
+ getStringCol("eventTime"), eventTimeZone)
+ val creationTimeZone = getOptStringCol("creationTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+ val creationTime: DateTime = new DateTime(
+ getStringCol("creationTime"), creationTimeZone)
+
+
+ Event(
+ eventId = eventId,
+ event = event,
+ entityType = entityType,
+ entityId = entityId,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ properties = properties,
+ eventTime = eventTime,
+ tags = Seq(),
+ prId = prId,
+ creationTime = creationTime
+ )
+ }
+
+ def eventToPut(event: Event, appId: Int): Map[String, Any] = {
+ Map(
+ "eventId" -> event.eventId,
+ "event" -> event.event,
+ "entityType" -> event.entityType,
+ "entityId" -> event.entityId,
+ "targetEntityType" -> event.targetEntityType,
+ "targetEntityId" -> event.targetEntityId,
+ "properties" -> event.properties.toJObject,
+ "eventTime" -> event.eventTime.toString,
+ "tags" -> event.tags,
+ "prId" -> event.prId,
+ "creationTime" -> event.creationTime.toString
+ )
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
new file mode 100644
index 0000000..fdd370a
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters._
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.LEvents
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.client.RestClient
+import org.joda.time.DateTime
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+import org.json4s.ext.JodaTimeSerializers
+
+import grizzled.slf4j.Logging
+import org.elasticsearch.client.ResponseException
+import org.apache.http.entity.StringEntity
+
+class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: String)
+ extends LEvents with Logging {
+ implicit val formats = DefaultFormats.lossless ++ JodaTimeSerializers.all
+ private val seq = new ESSequences(client, config, index)
+ private val seqName = "events"
+
+ def getEsType(appId: Int, channelId: Option[Int] = None): String = {
+ channelId.map { ch =>
+ s"${appId}_${ch}"
+ }.getOrElse {
+ s"${appId}"
+ }
+ }
+
+ override def init(appId: Int, channelId: Option[Int] = None): Boolean = {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val json =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)) ~
+ ("properties" ->
+ ("name" -> ("type" -> "keyword")) ~
+ ("eventId" -> ("type" -> "keyword")) ~
+ ("event" -> ("type" -> "keyword")) ~
+ ("entityType" -> ("type" -> "keyword")) ~
+ ("entityId" -> ("type" -> "keyword")) ~
+ ("targetEntityType" -> ("type" -> "keyword")) ~
+ ("targetEntityId" -> ("type" -> "keyword")) ~
+ ("properties" ->
+ ("type" -> "nested") ~
+ ("properties" ->
+ ("fields" -> ("type" -> "nested") ~
+ ("properties" ->
+ ("user" -> ("type" -> "long")) ~
+ ("num" -> ("type" -> "long")))))) ~
+ ("eventTime" -> ("type" -> "date")) ~
+ ("tags" -> ("type" -> "keyword")) ~
+ ("prId" -> ("type" -> "keyword")) ~
+ ("creationTime" -> ("type" -> "date"))))
+ ESUtils.createMapping(restClient, index, estype, compact(render(json)))
+ } finally {
+ restClient.close()
+ }
+ true
+ }
+
+ override def remove(appId: Int, channelId: Option[Int] = None): Boolean = {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("match_all" -> List.empty))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
+ restClient.performRequest(
+ "POST",
+ s"/$index/$estype/_delete_by_query",
+ Map("refresh" -> "true").asJava,
+ entity).getStatusLine.getStatusCode match {
+ case 200 => true
+ case _ =>
+ error(s"Failed to remove $index/$estype")
+ false
+ }
+ } catch {
+ case e: Exception =>
+ error(s"Failed to remove $index/$estype", e)
+ false
+ } finally {
+ restClient.close()
+ }
+ }
+
+ override def close(): Unit = {
+ // nothing
+ }
+
+ override def futureInsert(
+ event: Event,
+ appId: Int,
+ channelId: Option[Int])(implicit ec: ExecutionContext): Future[String] = {
+ Future {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ val id = event.eventId.getOrElse {
+ var roll = seq.genNext(seqName)
+ while (exists(restClient, estype, roll)) roll = seq.genNext(seqName)
+ roll.toString
+ }
+ val json = write(event.copy(eventId = Some(id)))
+ val entity = new NStringEntity(json, ContentType.APPLICATION_JSON);
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$id",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" => id
+ case "updated" => id
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype/$id")
+ ""
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype/<id>", e)
+ ""
+ } finally {
+ restClient.close()
+ }
+ }
+ }
+
+ private def exists(restClient: RestClient, estype: String, id: Int): Boolean = {
+ try {
+ restClient.performRequest(
+ "GET",
+ s"/$index/$estype/$id",
+ Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
+ case 200 => true
+ case _ => false
+ }
+ } catch {
+ case e: ResponseException =>
+ e.getResponse.getStatusLine.getStatusCode match {
+ case 404 => false
+ case _ =>
+ error(s"Failed to access to /$index/$estype/$id", e)
+ false
+ }
+ case e: IOException =>
+ error(s"Failed to access to $index/$estype/$id", e)
+ false
+ }
+ }
+
+ override def futureGet(
+ eventId: String,
+ appId: Int,
+ channelId: Option[Int])(implicit ec: ExecutionContext): Future[Option[Event]] = {
+ Future {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("eventId" -> eventId)))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/_search",
+ Map.empty[String, String].asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ (jsonResponse \ "hits" \ "total").extract[Long] match {
+ case 0 => None
+ case _ =>
+ val results = (jsonResponse \ "hits" \ "hits").extract[Seq[JValue]]
+ val result = (results.head \ "_source").extract[Event]
+ Some(result)
+ }
+ } catch {
+ case e: IOException =>
+ error("Failed to access to /$index/$estype/_search", e)
+ None
+ } finally {
+ restClient.close()
+ }
+ }
+ }
+
+ override def futureDelete(
+ eventId: String,
+ appId: Int,
+ channelId: Option[Int])(implicit ec: ExecutionContext): Future[Boolean] = {
+ Future {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("eventId" -> eventId)))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/_delete_by_query",
+ Map("refresh" -> "true").asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "deleted" => true
+ case _ =>
+ error(s"[$result] Failed to update $index/$estype:$eventId")
+ false
+ }
+ } catch {
+ case e: IOException =>
+ error(s"Failed to update $index/$estype:$eventId", e)
+ false
+ } finally {
+ restClient.close()
+ }
+ }
+ }
+
+ override def futureFind(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None,
+ limit: Option[Int] = None,
+ reversed: Option[Boolean] = None)
+ (implicit ec: ExecutionContext): Future[Iterator[Event]] = {
+ Future {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ val query = ESUtils.createEventQuery(
+ startTime, untilTime, entityType, entityId,
+ eventNames, targetEntityType, targetEntityId, reversed)
+ limit.getOrElse(20) match {
+ case -1 => ESUtils.getAll[Event](restClient, index, estype, query).toIterator
+ case size => ESUtils.get[Event](restClient, index, estype, query, size).toIterator
+ }
+ } catch {
+ case e: IOException =>
+ error(e.getMessage)
+ Iterator[Event]()
+ } finally {
+ restClient.close()
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
new file mode 100644
index 0000000..390e78c
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESPEvents.scala
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.io.MapWritable
+import org.apache.hadoop.io.Text
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.PEvents
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.elasticsearch.client.RestClient
+import org.elasticsearch.hadoop.mr.EsInputFormat
+import org.elasticsearch.spark._
+import org.joda.time.DateTime
+import java.io.IOException
+import org.apache.http.util.EntityUtils
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.entity.ContentType
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.ext.JodaTimeSerializers
+
+
+class ESPEvents(client: ESClient, config: StorageClientConfig, index: String)
+ extends PEvents {
+ implicit val formats = DefaultFormats.lossless ++ JodaTimeSerializers.all
+
+ def getEsType(appId: Int, channelId: Option[Int] = None): String = {
+ channelId.map { ch =>
+ s"${appId}_${ch}"
+ }.getOrElse {
+ s"${appId}"
+ }
+ }
+
+ def getESNodes(): String = {
+ val hosts = config.properties.get("HOSTS").
+ map(_.split(",").toSeq).getOrElse(Seq("localhost"))
+ val ports = config.properties.get("PORTS").
+ map(_.split(",").toSeq.map(_.toInt)).getOrElse(Seq(9200))
+ (hosts, ports).zipped.map(
+ (h, p) => s"$h:$p").mkString(",")
+ }
+
+ override def find(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None)(sc: SparkContext): RDD[Event] = {
+
+ val query = ESUtils.createEventQuery(
+ startTime, untilTime, entityType, entityId,
+ eventNames, targetEntityType, targetEntityId, None)
+
+ val estype = getEsType(appId, channelId)
+ val conf = new Configuration()
+ conf.set("es.resource", s"$index/$estype")
+ conf.set("es.query", query)
+ conf.set("es.nodes", getESNodes())
+
+ val rdd = sc.newAPIHadoopRDD(conf, classOf[EsInputFormat[Text, MapWritable]],
+ classOf[Text], classOf[MapWritable]).map {
+ case (key, doc) => {
+ ESEventsUtil.resultToEvent(key, doc, appId)
+ }
+ }
+
+ rdd
+ }
+
+ override def write(
+ events: RDD[Event],
+ appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+ val estype = getEsType(appId, channelId)
+ val conf = Map("es.resource" -> s"$index/$estype", "es.nodes" -> getESNodes())
+ events.map { event =>
+ ESEventsUtil.eventToPut(event, appId)
+ }.saveToEs(conf)
+ }
+
+ override def delete(
+ eventIds: RDD[String],
+ appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+ val estype = getEsType(appId, channelId)
+ val restClient = client.open()
+ try {
+ eventIds.foreachPartition { iter =>
+ iter.foreach { eventId =>
+ try {
+ val json =
+ ("query" ->
+ ("term" ->
+ ("eventId" -> eventId)))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/_delete_by_query",
+ Map("refresh" -> "true").asJava)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "deleted" => true
+ case _ =>
+ logger.error(s"[$result] Failed to update $index/$estype:$eventId")
+ false
+ }
+ } catch {
+ case e: IOException =>
+ logger.error(s"Failed to update $index/$estype:$eventId", e)
+ false
+ }
+ }
+ }
+ } finally {
+ restClient.close()
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
new file mode 100644
index 0000000..e5264ae
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import java.io.IOException
+
+import scala.collection.JavaConverters._
+
+import org.apache.http.Header
+import org.apache.http.entity.ContentType
+import org.apache.http.nio.entity.NStringEntity
+import org.apache.http.util.EntityUtils
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.write
+
+import grizzled.slf4j.Logging
+
+class ESSequences(client: ESClient, config: StorageClientConfig, index: String) extends Logging {
+ implicit val formats = DefaultFormats
+ private val estype = "sequences"
+
+ val restClient = client.open()
+ try {
+ ESUtils.createIndex(restClient, index)
+ val mappingJson =
+ (estype ->
+ ("_all" -> ("enabled" -> 0)))
+ ESUtils.createMapping(restClient, index, estype, compact(render(mappingJson)))
+ } finally {
+ restClient.close()
+ }
+
+ def genNext(name: String): Int = {
+ val restClient = client.open()
+ try {
+ val entity = new NStringEntity(write("n" -> name), ContentType.APPLICATION_JSON)
+ val response = restClient.performRequest(
+ "POST",
+ s"/$index/$estype/$name",
+ Map("refresh" -> "true").asJava,
+ entity)
+ val jsonResponse = parse(EntityUtils.toString(response.getEntity))
+ val result = (jsonResponse \ "result").extract[String]
+ result match {
+ case "created" =>
+ (jsonResponse \ "_version").extract[Int]
+ case "updated" =>
+ (jsonResponse \ "_version").extract[Int]
+ case _ =>
+ throw new IllegalStateException(s"[$result] Failed to update $index/$estype/$name")
+ }
+ } catch {
+ case e: IOException =>
+ throw new StorageClientException(s"Failed to update $index/$estype/$name", e)
+ } finally {
+ restClient.close()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
new file mode 100644
index 0000000..72f4dd6
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+
+import org.apache.http.entity.ContentType
+import org.apache.http.entity.StringEntity
+import org.apache.http.nio.entity.NStringEntity
+import org.elasticsearch.client.RestClient
+import org.json4s._
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.apache.http.util.EntityUtils
+import org.joda.time.DateTime
+import org.joda.time.format.DateTimeFormat
+import org.joda.time.DateTimeZone
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.http.HttpHost
+
+object ESUtils {
+ val scrollLife = "1m"
+
+ def get[T: Manifest](
+ client: RestClient,
+ index: String,
+ estype: String,
+ query: String,
+ size: Int)(
+ implicit formats: Formats): Seq[T] = {
+ val response = client.performRequest(
+ "POST",
+ s"/$index/$estype/_search",
+ Map("size" -> s"${size}"),
+ new StringEntity(query))
+ val responseJValue = parse(EntityUtils.toString(response.getEntity))
+ val hits = (responseJValue \ "hits" \ "hits").extract[Seq[JValue]]
+ hits.map(h => (h \ "_source").extract[T])
+ }
+
+ def getAll[T: Manifest](
+ client: RestClient,
+ index: String,
+ estype: String,
+ query: String)(
+ implicit formats: Formats): Seq[T] = {
+
+ @scala.annotation.tailrec
+ def scroll(scrollId: String, hits: Seq[JValue], results: Seq[T]): Seq[T] = {
+ if (hits.isEmpty) results
+ else {
+ val json = ("scroll" -> scrollLife) ~ ("scroll_id" -> scrollId)
+ val scrollBody = new StringEntity(compact(render(json)))
+ val response = client.performRequest(
+ "POST",
+ "/_search/scroll",
+ Map[String, String](),
+ scrollBody)
+ val responseJValue = parse(EntityUtils.toString(response.getEntity))
+ scroll((responseJValue \ "_scroll_id").extract[String],
+ (responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
+ hits.map(h => (h \ "_source").extract[T]) ++ results)
+ }
+ }
+
+ val response = client.performRequest(
+ "POST",
+ s"/$index/$estype/_search",
+ Map("scroll" -> scrollLife),
+ new StringEntity(query))
+ val responseJValue = parse(EntityUtils.toString(response.getEntity))
+ scroll((responseJValue \ "_scroll_id").extract[String],
+ (responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
+ Nil)
+ }
+
+ def createIndex(
+ client: RestClient,
+ index: String): Unit = {
+ client.performRequest(
+ "HEAD",
+ s"/$index",
+ Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
+ case 404 =>
+ client.performRequest(
+ "PUT",
+ s"/$index",
+ Map.empty[String, String].asJava)
+ case 200 =>
+ case _ =>
+ throw new IllegalStateException(s"/$index is invalid.")
+ }
+ }
+
+ def createMapping(
+ client: RestClient,
+ index: String,
+ estype: String,
+ json: String): Unit = {
+ client.performRequest(
+ "HEAD",
+ s"/$index/_mapping/$estype",
+ Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
+ case 404 =>
+ val entity = new NStringEntity(json, ContentType.APPLICATION_JSON)
+ client.performRequest(
+ "PUT",
+ s"/$index/_mapping/$estype",
+ Map.empty[String, String].asJava,
+ entity)
+ case 200 =>
+ case _ =>
+ throw new IllegalStateException(s"/$index/$estype is invalid: $json")
+ }
+ }
+
+ def createEventQuery(
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None,
+ reversed: Option[Boolean] = None): String = {
+ val mustQueries = Seq(
+ startTime.map(x => {
+ val v = DateTimeFormat
+ .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").print(x.withZone(DateTimeZone.UTC))
+ s"""{"range":{"eventTime":{"gte":"${v}"}}}"""
+ }),
+ untilTime.map(x => {
+ val v = DateTimeFormat
+ .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").print(x.withZone(DateTimeZone.UTC))
+ s"""{"range":{"eventTime":{"lt":"${v}"}}}"""
+ }),
+ entityType.map(x => s"""{"term":{"entityType":"${x}"}}"""),
+ entityId.map(x => s"""{"term":{"entityId":"${x}"}}"""),
+ targetEntityType.flatMap(xx => xx.map(x => s"""{"term":{"targetEntityType":"${x}"}}""")),
+ targetEntityId.flatMap(xx => xx.map(x => s"""{"term":{"targetEntityId":"${x}"}}""")),
+ eventNames
+ .map { xx => xx.map(x => "\"%s\"".format(x)) }
+ .map(x => s"""{"terms":{"event":[${x.mkString(",")}]}}""")).flatten.mkString(",")
+ val query = mustQueries.isEmpty match {
+ case true => """query":{"match_all":{}}"""
+ case _ => s"""query":{"bool":{"must":[${mustQueries}]}}"""
+ }
+ val sortOrder = reversed.map(x => x match {
+ case true => "desc"
+ case _ => "asc"
+ }).getOrElse("asc")
+ s"""{
+ |"${query},
+ |"sort":[{"eventTime":{"order":"${sortOrder}"}}]
+ |}""".stripMargin
+ }
+
+ def getHttpHosts(config: StorageClientConfig): Seq[HttpHost] = {
+ val hosts = config.properties.get("HOSTS").
+ map(_.split(",").toSeq).getOrElse(Seq("localhost"))
+ val ports = config.properties.get("PORTS").
+ map(_.split(",").toSeq.map(_.toInt)).getOrElse(Seq(9200))
+ val schemes = config.properties.get("SCHEMES").
+ map(_.split(",").toSeq).getOrElse(Seq("http"))
+ (hosts, ports, schemes).zipped.map((h, p, s) => new HttpHost(h, p, s))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
new file mode 100644
index 0000000..647d180
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import org.apache.http.HttpHost
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+import org.elasticsearch.client.RestClient
+
+import grizzled.slf4j.Logging
+
+case class ESClient(hosts: Seq[HttpHost]) {
+ def open(): RestClient = {
+ try {
+ RestClient.builder(hosts: _*).build()
+ } catch {
+ case e: Throwable =>
+ throw new StorageClientException(e.getMessage, e)
+ }
+ }
+}
+
+class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
+ with Logging {
+ override val prefix = "ES"
+
+ val client = ESClient(ESUtils.getHttpHosts(config))
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
new file mode 100644
index 0000000..fdc3b48
--- /dev/null
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** Elasticsearch implementation of storage traits, supporting meta data only
+ *
+ * @group Implementation
+ */
+package elasticsearch {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/test/resources/application.conf b/storage/elasticsearch/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/elasticsearch/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/.gitignore
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/.gitignore b/storage/elasticsearch1/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/elasticsearch1/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
new file mode 100644
index 0000000..8c29b84
--- /dev/null
+++ b/storage/elasticsearch1/build.sbt
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-elasticsearch1"
+
+elasticsearchVersion := "1.7.3"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.elasticsearch" % "elasticsearch" % elasticsearchVersion.value,
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "extra" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
+
[46/50] [abbrv] incubator-predictionio git commit: Cross building
Maven artifacts
Posted by do...@apache.org.
Cross building Maven artifacts
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/070f1794
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/070f1794
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/070f1794
Branch: refs/heads/livedoc
Commit: 070f1794d790bc312b390df1ff2379d44d1ac0a1
Parents: d5ff55f
Author: Donald Szeto <do...@apache.org>
Authored: Tue Apr 4 09:54:39 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Apr 4 09:54:39 2017 -0700
----------------------------------------------------------------------
build.sbt | 2 +-
storage/elasticsearch/build.sbt | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/070f1794/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 208bde7..26ba0da 100644
--- a/build.sbt
+++ b/build.sbt
@@ -64,7 +64,7 @@ javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7",
"-Xlint:deprecation", "-Xlint:unchecked")
// Ignore differentiation of Spark patch levels
-sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "1.6.3")
+sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", (if (scalaBinaryVersion.value == "2.10") "1.6.3" else "2.1.0"))
sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/070f1794/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 50292c5..7500d1e 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -21,6 +21,8 @@ name := "apache-predictionio-data-elasticsearch"
elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13")
+elasticsearchVersion := (if (majorVersion(elasticsearchVersion.value) < 5) "5.2.2" else elasticsearchVersion.value)
+
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
[10/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
new file mode 100644
index 0000000..b9ec957
--- /dev/null
+++ b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.localfs
+
+import java.io.File
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+
+class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
+ with Logging {
+ override val prefix = "LocalFS"
+ val f = new File(
+ config.properties.getOrElse("PATH", config.properties("HOSTS")))
+ if (f.exists) {
+ if (!f.isDirectory) throw new StorageClientException(
+ s"${f} already exists but it is not a directory!",
+ null)
+ if (!f.canWrite) throw new StorageClientException(
+ s"${f} already exists but it is not writable!",
+ null)
+ } else {
+ if (!f.mkdirs) throw new StorageClientException(
+ s"${f} does not exist and automatic creation failed!",
+ null)
+ }
+ val client = f
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
----------------------------------------------------------------------
diff --git a/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
new file mode 100644
index 0000000..554ab26
--- /dev/null
+++ b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** Local file system implementation of storage traits, supporting model data only
+ *
+ * @group Implementation
+ */
+package object localfs {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/localfs/src/test/resources/application.conf b/storage/localfs/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/localfs/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index d1d048a..94f5688 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -17,11 +17,13 @@
FROM predictionio/pio
-ENV SPARK_VERSION 1.4.0
-ENV ELASTICSEARCH_VERSION 1.4.4
+ENV SPARK_VERSION 1.6.3
+ENV ELASTICSEARCH_VERSION 5.2.1
ENV HBASE_VERSION 1.0.0
ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
+# WORKAROUND: es-hadoop stops on RDD#take(1)
+RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
@@ -31,6 +33,10 @@ COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh
COPY docker-files/pgpass /root/.pgpass
RUN chmod 600 /root/.pgpass
+# Python
+RUN pip install python-dateutil
+RUN pip install pytz
+
# Default repositories setup
ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index dd95168..459b929 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -17,9 +17,9 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ ! -f $DIR/docker-files/spark-1.4.0-bin-hadoop2.6.tgz ]; then
- wget http://d3kbcqa49mib13.cloudfront.net/spark-1.4.0-bin-hadoop2.6.tgz
- mv spark-1.4.0-bin-hadoop2.6.tgz $DIR/docker-files/
+if [ ! -f $DIR/docker-files/spark-1.6.3-bin-hadoop2.6.tgz ]; then
+ wget http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz
+ mv spark-1.6.3-bin-hadoop2.6.tgz $DIR/docker-files/
fi
if [ ! -f $DIR/docker-files/postgresql-9.4-1204.jdbc41.jar ]; then
@@ -33,6 +33,8 @@ pushd $DIR/..
sbt/sbt clean
mkdir assembly
cp dist/lib/*.jar assembly/
+mkdir -p lib/spark
+cp dist/lib/spark/*.jar lib/spark
docker build -t predictionio/pio .
popd
docker build -t predictionio/pio-testing $DIR
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 3939a0b..b556f7b 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,7 @@
version: "2"
services:
elasticsearch:
- image: elasticsearch:1-alpine
+ image: elasticsearch:5-alpine
hbase:
image: harisekhon/hbase:1.0
postgres:
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/docker-files/env-conf/pio-env.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh
index e1076ba..0acf3a7 100644
--- a/tests/docker-files/env-conf/pio-env.sh
+++ b/tests/docker-files/env-conf/pio-env.sh
@@ -87,7 +87,8 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
-PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
+PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
+PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
# Local File System Example
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/pio_tests/scenarios/eventserver_test.py
----------------------------------------------------------------------
diff --git a/tests/pio_tests/scenarios/eventserver_test.py b/tests/pio_tests/scenarios/eventserver_test.py
index c09e815..6f29876 100644
--- a/tests/pio_tests/scenarios/eventserver_test.py
+++ b/tests/pio_tests/scenarios/eventserver_test.py
@@ -19,6 +19,8 @@ import unittest
import requests
import json
import argparse
+import dateutil.parser
+import pytz
from subprocess import Popen
from utils import AppEngine, pjoin
from pio_tests.integration import BaseTestCase, AppContext
@@ -155,7 +157,8 @@ class EventserverTest(BaseTestCase):
'reversed': 'true' }
r = self.app.get_events(params=params)
self.assertEqual(5, len(r.json()))
- self.assertEqual('2014-11-05T09:39:45.618-08:00', r.json()[0]['eventTime'])
+ event_time = dateutil.parser.parse(r.json()[0]['eventTime']).astimezone(pytz.utc)
+ self.assertEqual('2014-11-05 17:39:45.618000+00:00', str(event_time))
def tearDown(self):
self.log.info("Deleting all app data")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index 9f28d1c..fe07957 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -19,7 +19,7 @@
USAGE=$"Usage: run_docker <meta> <event> <model> <command>
Where:
meta = [PGSQL,ELASTICSEARCH]
- event = [PGSQL,HBASE]
+ event = [PGSQL,HBASE,ELASTICSEARCH]
model = [PGSQL,LOCALFS,HDFS]
command = command to run in the container"
@@ -30,7 +30,7 @@ fi
META="$1"
shift
-if ! [[ "$1" =~ ^(PGSQL|HBASE)$ ]]; then
+if ! [[ "$1" =~ ^(PGSQL|HBASE|ELASTICSEARCH)$ ]]; then
echo "$USAGE"
exit 1
fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
index 6c56615..7d04c07 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -102,6 +102,13 @@ object Common extends EitherLogging {
if (targetFiles.size > 0) targetFiles else libFiles
}
+ def jarFilesForSpark(pioHome: String): Array[File] = {
+ def jarFilesAt(path: File): Array[File] = path.listFiles filter {
+ _.getName.toLowerCase.endsWith(".jar")
+ }
+ jarFilesAt(new File(pioHome, "lib/spark"))
+ }
+
def coreAssembly(pioHome: String): Expected[File] = {
val core = s"pio-assembly-${BuildInfo.version}.jar"
val coreDir =
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index 4f5a176..662dbbf 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -160,7 +160,8 @@ object Runner extends EitherLogging {
val sparkSubmitCommand =
Seq(Seq(sparkHome, "bin", "spark-submit").mkString(File.separator))
- val sparkSubmitJarsList = WorkflowUtils.thirdPartyJars ++ deployedJars
+ val sparkSubmitJarsList = WorkflowUtils.thirdPartyJars ++ deployedJars ++
+ Common.jarFilesForSpark(pioHome).map(_.toURI)
val sparkSubmitJars = if (sparkSubmitJarsList.nonEmpty) {
Seq("--jars", sparkSubmitJarsList.map(_.toString).mkString(","))
} else {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index 4656457..69a3924 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -297,7 +297,9 @@ object Engine extends EitherLogging {
val extraFiles = WorkflowUtils.thirdPartyConfFiles
val jarFiles = jarFilesForScala(engineDirPath)
jarFiles foreach { f => info(s"Found JAR: ${f.getName}") }
- val allJarFiles = jarFiles.map(_.getCanonicalPath)
+ val jarPluginFiles = jarFilesForSpark(pioHome)
+ jarPluginFiles foreach { f => info(s"Found JAR: ${f.getName}") }
+ val allJarFiles = jarFiles.map(_.getCanonicalPath) ++ jarPluginFiles.map(_.getCanonicalPath)
val cmd = s"${getSparkHome(sparkArgs.sparkHome)}/bin/spark-submit --jars " +
s"${allJarFiles.mkString(",")} " +
[43/50] [abbrv] incubator-predictionio git commit: PIO-49 has been
fixed by #352
Posted by do...@apache.org.
PIO-49 has been fixed by #352
Closes #336
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/ba46e2c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/ba46e2c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/ba46e2c2
Branch: refs/heads/livedoc
Commit: ba46e2c2a72657d632af9ef0e8ca3edf14276d65
Parents: d64b721
Author: Donald Szeto <do...@apache.org>
Authored: Mon Apr 3 10:51:12 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Apr 3 10:51:12 2017 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[48/50] [abbrv] incubator-predictionio git commit: Add PMC doc
Posted by do...@apache.org.
Add PMC doc
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/5a35586a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/5a35586a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/5a35586a
Branch: refs/heads/livedoc
Commit: 5a35586af46617a99200db4c8998b350605359c7
Parents: 9deca1a
Author: Donald Szeto <do...@apache.org>
Authored: Sun Apr 9 11:33:37 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Apr 9 11:33:37 2017 -0700
----------------------------------------------------------------------
PMC.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 82 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/5a35586a/PMC.md
----------------------------------------------------------------------
diff --git a/PMC.md b/PMC.md
new file mode 100644
index 0000000..3609f34
--- /dev/null
+++ b/PMC.md
@@ -0,0 +1,82 @@
+# Project Management Committee Documentation
+
+## Release Procedure
+
+1. Generate code signing key if you do not already have one for Apache. Refer to
+http://apache.org/dev/openpgp.html#generate-key on how to generate a strong code
+signing key.
+2. Add your public key to the `KEYS` file at the root of the source code tree.
+3. Create a new release branch, with version bumped to the next release version.
+ 1. `git checkout -b release/0.11.0`
+ 2. Replace all `0.11.0-SNAPSHOT` in the code tree to `0.11.0-incubating`.
+ 3. `git commit -am "Prepare 0.11.0-incubating-rc1"`
+4. If you have not done so, use SVN to checkout
+https://dist.apache.org/repos/dist/dev/incubator/predictionio. This is the area
+for staging release candidates for voting.
+ 1. `svn co https://dist.apache.org/repos/dist/dev/incubator/predictionio`
+5. Package a clean tarball for staging a release candidate.
+ 1. `git archive --format tar v0.11.0-incubating-rc1 >
+ ../apache-predictionio-0.11.0-incubating-rc1.tar`
+ 2. `cd ..; gzip apache-predictionio-0.11.0-incubating-rc1.tar`
+ 3. Go to the parent directory and do `tar zcvf
+ apache-predictionio-0.11.0-incubating-rc1.tar.gz predictionio`
+6. Generate detached signature for the release candidate.
+(http://apache.org/dev/release-signing.html#openpgp-ascii-detach-sig)
+ 1. `gpg --armor --output apache-predictionio-0.11.0-incubating-rc1.tar.gz.asc
+ --detach-sig apache-predictionio-0.11.0-incubating-rc1.tar.gz`
+7. Generate MD5 and SHA512 checksums for the release candidate.
+ 1. `gpg --print-md MD5 apache-predictionio-0.11.0-incubating-rc1.tar.gz >
+ apache-predictionio-0.11.0-incubating-rc1.tar.gz.md5`
+ 2. `gpg --print-md SHA512 apache-predictionio-0.11.0-incubating-rc1.tar.gz >
+ apache-predictionio-0.11.0-incubating-rc1.tar.gz.sha512`
+8. Create a subdirectory at the SVN staging area. The area should have a `KEYS` file.
+ 1. `mkdir apache-predictionio-0.11.0-incubating-rc1`
+ 2. `cp apache-predictionio-0.11.0-incubating-rc1.*
+ apache-predictionio-0.11.0-incubating-rc1`
+9. If you have updated the `KEYS` file, also copy that to the staging area.
+10. `svn commit`
+11. Set up credentials with Apache Nexus using the SBT Sonatype plugin. Put this
+in `~/.sbt/0.13/sonatype.sbt`. You can generate username and password tokens
+from ASF's Nexus instance.
+
+ ```
+ publishTo := {
+ val nexus = "https://repository.apache.org/"
+ if (isSnapshot.value)
+ Some("snapshots" at nexus + "content/repositories/snapshots")
+ else
+ Some("releases" at nexus + "service/local/staging/deploy/maven2")
+ }
+
+ credentials += Credentials("Sonatype Nexus Repository Manager", "repository.apache.org", "username_token", "password_token")
+ ```
+
+12. `sbt/sbt +publishSigned +storage/publishSigned
++dataElasticsearch/publishSigned` then close the staged repository on Apache
+Nexus.
+13. Wait for Travis to pass build on the release branch.
+14. Tag the release branch with a rc tag, e.g. `0.11.0-incubating-rc1`.
+15. Send out e-mail for voting on PredictionIO dev mailing list.
+
+ ```
+ Subject: [VOTE] Apache PredictionIO (incubating) 0.11.0 Release (RC1)
+
+ This is the vote for 0.11.0 of Apache PredictionIO (incubating).
+
+ The vote will run for at least 72 hours and will close on Apr 7th, 2017.
+
+ The release candidate artifacts can be downloaded here: https://dist.apache.org/repos/dist/dev/incubator/predictionio/0.11.0-incubating-rc1/
+
+ Test results of RC5 can be found here: https://travis-ci.org/apache/incubator-predictionio/builds/xxx
+
+ Maven artifacts are built from the release candidate artifacts above, and are provided as convenience for testing with engine templates. The Maven artifacts are provided at the Maven staging repo here: https://repository.apache.org/content/repositories/orgapachepredictionio-nnnn/
+
+ All JIRAs completed for this release are tagged with 'FixVersion = 0.10.0'. You can view them here: https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320420&version=12337844
+
+ The artifacts have been signed with Key : YOUR_KEY_ID
+
+ Please vote accordingly:
+
+ [ ] +1, accept RC as the official 0.10.0 release
+ [ ] -1, do not accept RC as the official 0.10.0 release because...
+ ```
[28/50] [abbrv] incubator-predictionio git commit: Move storage
depended tests to each storage project
Posted by do...@apache.org.
Move storage depended tests to each storage project
Closes #362
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/647b480b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/647b480b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/647b480b
Branch: refs/heads/livedoc
Commit: 647b480b0f0f693ec1b4d04dc9a2cf1492f0a0fe
Parents: 606be41
Author: Naoki Takezoe <ta...@gmail.com>
Authored: Wed Mar 15 09:42:30 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Wed Mar 15 09:42:30 2017 -0700
----------------------------------------------------------------------
.../predictionio/data/storage/LEventsSpec.scala | 248 -----------------
.../predictionio/data/storage/PEventsSpec.scala | 213 ---------------
.../data/storage/StorageTestUtils.scala | 45 ----
.../data/storage/hbase/LEventsSpec.scala | 239 +++++++++++++++++
.../data/storage/hbase/PEventsSpec.scala | 192 +++++++++++++
.../data/storage/hbase/StorageTestUtils.scala | 40 +++
.../data/storage/hbase/TestEvents.scala | 266 +++++++++++++++++++
.../data/storage/jdbc/LEventsSpec.scala | 236 ++++++++++++++++
.../data/storage/jdbc/PEventsSpec.scala | 193 ++++++++++++++
.../data/storage/jdbc/StorageTestUtils.scala | 29 ++
.../data/storage/jdbc/TestEvents.scala | 266 +++++++++++++++++++
11 files changed, 1461 insertions(+), 506 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/data/src/test/scala/org/apache/predictionio/data/storage/LEventsSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/LEventsSpec.scala b/data/src/test/scala/org/apache/predictionio/data/storage/LEventsSpec.scala
deleted file mode 100644
index 3938072..0000000
--- a/data/src/test/scala/org/apache/predictionio/data/storage/LEventsSpec.scala
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-import org.specs2._
-import org.specs2.specification.Step
-
-class LEventsSpec extends Specification with TestEvents {
- def is = s2"""
-
- PredictionIO Storage LEvents Specification
-
- Events can be implemented by:
- - HBLEvents ${hbEvents}
- - JDBCLEvents ${jdbcLEvents}
-
- """
-
- def hbEvents = sequential ^ s2"""
-
- HBLEvents should
- - behave like any LEvents implementation ${events(hbDO)}
- - (table cleanup) ${Step(StorageTestUtils.dropHBaseNamespace(dbName))}
-
- """
-
- def jdbcLEvents = sequential ^ s2"""
-
- JDBCLEvents should
- - behave like any LEvents implementation ${events(jdbcDO)}
-
- """
-
- val appId = 1
-
- def events(eventClient: LEvents) = sequential ^ s2"""
-
- init default ${initDefault(eventClient)}
- insert 3 test events and get back by event ID ${insertAndGetEvents(eventClient)}
- insert 3 test events with timezone and get back by event ID ${insertAndGetTimezone(eventClient)}
- insert and delete by ID ${insertAndDelete(eventClient)}
- insert test user events ${insertTestUserEvents(eventClient)}
- find user events ${findUserEvents(eventClient)}
- aggregate user properties ${aggregateUserProperties(eventClient)}
- aggregate one user properties ${aggregateOneUserProperties(eventClient)}
- aggregate non-existent user properties ${aggregateNonExistentUserProperties(eventClient)}
- init channel ${initChannel(eventClient)}
- insert 2 events to channel ${insertChannel(eventClient)}
- insert 1 event to channel and delete by ID ${insertAndDeleteChannel(eventClient)}
- find events from channel ${findChannel(eventClient)}
- remove default ${removeDefault(eventClient)}
- remove channel ${removeChannel(eventClient)}
-
- """
-
- val dbName = "test_pio_storage_events_" + hashCode
- def hbDO = Storage.getDataObject[LEvents](
- StorageTestUtils.hbaseSourceName,
- dbName
- )
-
- def jdbcDO = Storage.getDataObject[LEvents](StorageTestUtils.jdbcSourceName, dbName)
-
- def initDefault(eventClient: LEvents) = {
- eventClient.init(appId)
- }
-
- def insertAndGetEvents(eventClient: LEvents) = {
-
- // events from TestEvents trait
- val listOfEvents = List(r1,r2,r3)
-
- val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
-
- val insertedEventId: List[String] = insertResp
-
- val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
- .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
-
- val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
-
- val getEvents = getResp
-
- insertedEvent must containTheSameElementsAs(getEvents)
- }
-
- def insertAndGetTimezone(eventClient: LEvents) = {
- val listOfEvents = List(tz1, tz2, tz3)
-
- val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
-
- val insertedEventId: List[String] = insertResp
-
- val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
- .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
-
- val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
-
- val getEvents = getResp
-
- insertedEvent must containTheSameElementsAs(getEvents)
- }
-
- def insertAndDelete(eventClient: LEvents) = {
- val eventId = eventClient.insert(r2, appId)
-
- val resultBefore = eventClient.get(eventId, appId)
-
- val expectedBefore = r2.copy(eventId = Some(eventId))
-
- val deleteStatus = eventClient.delete(eventId, appId)
-
- val resultAfter = eventClient.get(eventId, appId)
-
- (resultBefore must beEqualTo(Some(expectedBefore))) and
- (deleteStatus must beEqualTo(true)) and
- (resultAfter must beEqualTo(None))
- }
-
- def insertTestUserEvents(eventClient: LEvents) = {
- // events from TestEvents trait
- val listOfEvents = Vector(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
-
- listOfEvents.map{ eventClient.insert(_, appId) }
-
- success
- }
-
- def findUserEvents(eventClient: LEvents) = {
-
- val results: List[Event] = eventClient.find(
- appId = appId,
- entityType = Some("user"))
- .toList
- .map(e => e.copy(eventId = None)) // ignore eventID
-
- // same events in insertTestUserEvents
- val expected = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
-
- results must containTheSameElementsAs(expected)
- }
-
- def aggregateUserProperties(eventClient: LEvents) = {
-
- val result: Map[String, PropertyMap] = eventClient.aggregateProperties(
- appId = appId,
- entityType = "user")
-
- val expected = Map(
- "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
- "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
- )
-
- result must beEqualTo(expected)
- }
-
- def aggregateOneUserProperties(eventClient: LEvents) = {
- val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
- appId = appId,
- entityType = "user",
- entityId = "u1")
-
- val expected = Some(PropertyMap(u1, u1BaseTime, u1LastTime))
-
- result must beEqualTo(expected)
- }
-
- def aggregateNonExistentUserProperties(eventClient: LEvents) = {
- val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
- appId = appId,
- entityType = "user",
- entityId = "u999999")
-
- result must beEqualTo(None)
- }
-
- val channelId = 12
-
- def initChannel(eventClient: LEvents) = {
- eventClient.init(appId, Some(channelId))
- }
-
- def insertChannel(eventClient: LEvents) = {
-
- // events from TestEvents trait
- val listOfEvents = List(r4,r5)
-
- listOfEvents.map( eventClient.insert(_, appId, Some(channelId)) )
-
- success
- }
-
- def insertAndDeleteChannel(eventClient: LEvents) = {
-
- val eventId = eventClient.insert(r2, appId, Some(channelId))
-
- val resultBefore = eventClient.get(eventId, appId, Some(channelId))
-
- val expectedBefore = r2.copy(eventId = Some(eventId))
-
- val deleteStatus = eventClient.delete(eventId, appId, Some(channelId))
-
- val resultAfter = eventClient.get(eventId, appId, Some(channelId))
-
- (resultBefore must beEqualTo(Some(expectedBefore))) and
- (deleteStatus must beEqualTo(true)) and
- (resultAfter must beEqualTo(None))
- }
-
- def findChannel(eventClient: LEvents) = {
-
- val results: List[Event] = eventClient.find(
- appId = appId,
- channelId = Some(channelId)
- )
- .toList
- .map(e => e.copy(eventId = None)) // ignore eventId
-
- // same events in insertChannel
- val expected = List(r4, r5)
-
- results must containTheSameElementsAs(expected)
- }
-
- def removeDefault(eventClient: LEvents) = {
- eventClient.remove(appId)
- }
-
- def removeChannel(eventClient: LEvents) = {
- eventClient.remove(appId, Some(channelId))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/data/src/test/scala/org/apache/predictionio/data/storage/PEventsSpec.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/PEventsSpec.scala b/data/src/test/scala/org/apache/predictionio/data/storage/PEventsSpec.scala
deleted file mode 100644
index ccd71a4..0000000
--- a/data/src/test/scala/org/apache/predictionio/data/storage/PEventsSpec.scala
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-import org.specs2._
-import org.specs2.specification.Step
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-class PEventsSpec extends Specification with TestEvents {
-
- System.clearProperty("spark.driver.port")
- System.clearProperty("spark.hostPort")
- val sc = new SparkContext("local[4]", "PEventAggregatorSpec test")
-
- val appId = 1
- val channelId = 6
- val dbName = "test_pio_storage_events_" + hashCode
-
- def hbLocal = Storage.getDataObject[LEvents](
- StorageTestUtils.hbaseSourceName,
- dbName
- )
-
- def hbPar = Storage.getDataObject[PEvents](
- StorageTestUtils.hbaseSourceName,
- dbName
- )
-
- def jdbcLocal = Storage.getDataObject[LEvents](
- StorageTestUtils.jdbcSourceName,
- dbName
- )
-
- def jdbcPar = Storage.getDataObject[PEvents](
- StorageTestUtils.jdbcSourceName,
- dbName
- )
-
- def stopSpark = {
- sc.stop()
- }
-
- def is = s2"""
-
- PredictionIO Storage PEvents Specification
-
- PEvents can be implemented by:
- - HBPEvents ${hbPEvents}
- - JDBCPEvents ${jdbcPEvents}
- - (stop Spark) ${Step(sc.stop())}
-
- """
-
- def hbPEvents = sequential ^ s2"""
-
- HBPEvents should
- - behave like any PEvents implementation ${events(hbLocal, hbPar)}
- - (table cleanup) ${Step(StorageTestUtils.dropHBaseNamespace(dbName))}
-
- """
-
- def jdbcPEvents = sequential ^ s2"""
-
- JDBCPEvents should
- - behave like any PEvents implementation ${events(jdbcLocal, jdbcPar)}
- - (table cleanup) ${Step(StorageTestUtils.dropJDBCTable(s"${dbName}_$appId"))}
- - (table cleanup) ${Step(StorageTestUtils.dropJDBCTable(s"${dbName}_${appId}_$channelId"))}
-
- """
-
- def events(localEventClient: LEvents, parEventClient: PEvents) = sequential ^ s2"""
-
- - (init test) ${initTest(localEventClient)}
- - (insert test events) ${insertTestEvents(localEventClient)}
- find in default ${find(parEventClient)}
- find in channel ${findChannel(parEventClient)}
- aggregate user properties in default ${aggregateUserProperties(parEventClient)}
- aggregate user properties in channel ${aggregateUserPropertiesChannel(parEventClient)}
- write to default ${write(parEventClient)}
- write to channel ${writeChannel(parEventClient)}
-
- """
-
- /* setup */
-
- // events from TestEvents trait
- val listOfEvents = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2, r1, r2)
- val listOfEventsChannel = List(u3e1, u3e2, u3e3, r3, r4)
-
- def initTest(localEventClient: LEvents) = {
- localEventClient.init(appId)
- localEventClient.init(appId, Some(channelId))
- }
-
- def insertTestEvents(localEventClient: LEvents) = {
- listOfEvents.map( localEventClient.insert(_, appId) )
- // insert to channel
- listOfEventsChannel.map( localEventClient.insert(_, appId, Some(channelId)) )
- success
- }
-
- /* following are tests */
-
- def find(parEventClient: PEvents) = {
- val resultRDD: RDD[Event] = parEventClient.find(
- appId = appId
- )(sc)
-
- val results = resultRDD.collect.toList
- .map {_.copy(eventId = None)} // ignore eventId
-
- results must containTheSameElementsAs(listOfEvents)
- }
-
- def findChannel(parEventClient: PEvents) = {
- val resultRDD: RDD[Event] = parEventClient.find(
- appId = appId,
- channelId = Some(channelId)
- )(sc)
-
- val results = resultRDD.collect.toList
- .map {_.copy(eventId = None)} // ignore eventId
-
- results must containTheSameElementsAs(listOfEventsChannel)
- }
-
- def aggregateUserProperties(parEventClient: PEvents) = {
- val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
- appId = appId,
- entityType = "user"
- )(sc)
- val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
-
- val expected = Map(
- "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
- "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
- )
-
- result must beEqualTo(expected)
- }
-
- def aggregateUserPropertiesChannel(parEventClient: PEvents) = {
- val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
- appId = appId,
- channelId = Some(channelId),
- entityType = "user"
- )(sc)
- val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
-
- val expected = Map(
- "u3" -> PropertyMap(u3, u3BaseTime, u3LastTime)
- )
-
- result must beEqualTo(expected)
- }
-
- def write(parEventClient: PEvents) = {
- val written = List(r5, r6)
- val writtenRDD = sc.parallelize(written)
- parEventClient.write(writtenRDD, appId)(sc)
-
- // read back
- val resultRDD = parEventClient.find(
- appId = appId
- )(sc)
-
- val results = resultRDD.collect.toList
- .map { _.copy(eventId = None)} // ignore eventId
-
- val expected = listOfEvents ++ written
-
- results must containTheSameElementsAs(expected)
- }
-
- def writeChannel(parEventClient: PEvents) = {
- val written = List(r1, r5, r6)
- val writtenRDD = sc.parallelize(written)
- parEventClient.write(writtenRDD, appId, Some(channelId))(sc)
-
- // read back
- val resultRDD = parEventClient.find(
- appId = appId,
- channelId = Some(channelId)
- )(sc)
-
- val results = resultRDD.collect.toList
- .map { _.copy(eventId = None)} // ignore eventId
-
- val expected = listOfEventsChannel ++ written
-
- results must containTheSameElementsAs(expected)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/data/src/test/scala/org/apache/predictionio/data/storage/StorageTestUtils.scala
----------------------------------------------------------------------
diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/StorageTestUtils.scala b/data/src/test/scala/org/apache/predictionio/data/storage/StorageTestUtils.scala
deleted file mode 100644
index 747076a..0000000
--- a/data/src/test/scala/org/apache/predictionio/data/storage/StorageTestUtils.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-import org.apache.predictionio.data.storage.hbase.HBLEvents
-import scalikejdbc._
-
-object StorageTestUtils {
- val hbaseSourceName = "HBASE"
- val jdbcSourceName = "PGSQL"
-
- def dropHBaseNamespace(namespace: String): Unit = {
- val eventDb = Storage.getDataObject[LEvents](hbaseSourceName, namespace)
- .asInstanceOf[HBLEvents]
- val admin = eventDb.client.admin
- val tableNames = admin.listTableNamesByNamespace(namespace)
- tableNames.foreach { name =>
- admin.disableTable(name)
- admin.deleteTable(name)
- }
-
- //Only empty namespaces (no tables) can be removed.
- admin.deleteNamespace(namespace)
- }
-
- def dropJDBCTable(table: String): Unit = DB autoCommit { implicit s =>
- SQL(s"drop table $table").execute().apply()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/LEventsSpec.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/LEventsSpec.scala b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/LEventsSpec.scala
new file mode 100644
index 0000000..c813ced
--- /dev/null
+++ b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/LEventsSpec.scala
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage.{Event, LEvents, PropertyMap, Storage}
+import org.specs2._
+import org.specs2.specification.Step
+
+class LEventsSpec extends Specification with TestEvents {
+ def is = s2"""
+
+ PredictionIO Storage LEvents Specification
+
+ Events can be implemented by:
+ - HBLEvents ${hbEvents}
+
+ """
+
+ def hbEvents = sequential ^ s2"""
+
+ HBLEvents should
+ - behave like any LEvents implementation ${events(hbDO)}
+ - (table cleanup) ${Step(StorageTestUtils.dropHBaseNamespace(dbName))}
+
+ """
+
+ val appId = 1
+
+ def events(eventClient: LEvents) = sequential ^ s2"""
+
+ init default ${initDefault(eventClient)}
+ insert 3 test events and get back by event ID ${insertAndGetEvents(eventClient)}
+ insert 3 test events with timezone and get back by event ID ${insertAndGetTimezone(eventClient)}
+ insert and delete by ID ${insertAndDelete(eventClient)}
+ insert test user events ${insertTestUserEvents(eventClient)}
+ find user events ${findUserEvents(eventClient)}
+ aggregate user properties ${aggregateUserProperties(eventClient)}
+ aggregate one user properties ${aggregateOneUserProperties(eventClient)}
+ aggregate non-existent user properties ${aggregateNonExistentUserProperties(eventClient)}
+ init channel ${initChannel(eventClient)}
+ insert 2 events to channel ${insertChannel(eventClient)}
+ insert 1 event to channel and delete by ID ${insertAndDeleteChannel(eventClient)}
+ find events from channel ${findChannel(eventClient)}
+ remove default ${removeDefault(eventClient)}
+ remove channel ${removeChannel(eventClient)}
+
+ """
+
+ val dbName = "test_pio_storage_events_" + hashCode
+ def hbDO = Storage.getDataObject[LEvents](
+ StorageTestUtils.hbaseSourceName,
+ dbName
+ )
+
+ def initDefault(eventClient: LEvents) = {
+ eventClient.init(appId)
+ }
+
+ def insertAndGetEvents(eventClient: LEvents) = {
+
+ // events from TestEvents trait
+ val listOfEvents = List(r1,r2,r3)
+
+ val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
+
+ val insertedEventId: List[String] = insertResp
+
+ val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
+ .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
+
+ val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
+
+ val getEvents = getResp
+
+ insertedEvent must containTheSameElementsAs(getEvents)
+ }
+
+ def insertAndGetTimezone(eventClient: LEvents) = {
+ val listOfEvents = List(tz1, tz2, tz3)
+
+ val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
+
+ val insertedEventId: List[String] = insertResp
+
+ val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
+ .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
+
+ val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
+
+ val getEvents = getResp
+
+ insertedEvent must containTheSameElementsAs(getEvents)
+ }
+
+ def insertAndDelete(eventClient: LEvents) = {
+ val eventId = eventClient.insert(r2, appId)
+
+ val resultBefore = eventClient.get(eventId, appId)
+
+ val expectedBefore = r2.copy(eventId = Some(eventId))
+
+ val deleteStatus = eventClient.delete(eventId, appId)
+
+ val resultAfter = eventClient.get(eventId, appId)
+
+ (resultBefore must beEqualTo(Some(expectedBefore))) and
+ (deleteStatus must beEqualTo(true)) and
+ (resultAfter must beEqualTo(None))
+ }
+
+ def insertTestUserEvents(eventClient: LEvents) = {
+ // events from TestEvents trait
+ val listOfEvents = Vector(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
+
+ listOfEvents.map{ eventClient.insert(_, appId) }
+
+ success
+ }
+
+ def findUserEvents(eventClient: LEvents) = {
+
+ val results: List[Event] = eventClient.find(
+ appId = appId,
+ entityType = Some("user"))
+ .toList
+ .map(e => e.copy(eventId = None)) // ignore eventID
+
+ // same events in insertTestUserEvents
+ val expected = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def aggregateUserProperties(eventClient: LEvents) = {
+
+ val result: Map[String, PropertyMap] = eventClient.aggregateProperties(
+ appId = appId,
+ entityType = "user")
+
+ val expected = Map(
+ "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
+ "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateOneUserProperties(eventClient: LEvents) = {
+ val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
+ appId = appId,
+ entityType = "user",
+ entityId = "u1")
+
+ val expected = Some(PropertyMap(u1, u1BaseTime, u1LastTime))
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateNonExistentUserProperties(eventClient: LEvents) = {
+ val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
+ appId = appId,
+ entityType = "user",
+ entityId = "u999999")
+
+ result must beEqualTo(None)
+ }
+
+ val channelId = 12
+
+ def initChannel(eventClient: LEvents) = {
+ eventClient.init(appId, Some(channelId))
+ }
+
+ def insertChannel(eventClient: LEvents) = {
+
+ // events from TestEvents trait
+ val listOfEvents = List(r4,r5)
+
+ listOfEvents.map( eventClient.insert(_, appId, Some(channelId)) )
+
+ success
+ }
+
+ def insertAndDeleteChannel(eventClient: LEvents) = {
+
+ val eventId = eventClient.insert(r2, appId, Some(channelId))
+
+ val resultBefore = eventClient.get(eventId, appId, Some(channelId))
+
+ val expectedBefore = r2.copy(eventId = Some(eventId))
+
+ val deleteStatus = eventClient.delete(eventId, appId, Some(channelId))
+
+ val resultAfter = eventClient.get(eventId, appId, Some(channelId))
+
+ (resultBefore must beEqualTo(Some(expectedBefore))) and
+ (deleteStatus must beEqualTo(true)) and
+ (resultAfter must beEqualTo(None))
+ }
+
+ def findChannel(eventClient: LEvents) = {
+
+ val results: List[Event] = eventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )
+ .toList
+ .map(e => e.copy(eventId = None)) // ignore eventId
+
+ // same events in insertChannel
+ val expected = List(r4, r5)
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def removeDefault(eventClient: LEvents) = {
+ eventClient.remove(appId)
+ }
+
+ def removeChannel(eventClient: LEvents) = {
+ eventClient.remove(appId, Some(channelId))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/PEventsSpec.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/PEventsSpec.scala b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/PEventsSpec.scala
new file mode 100644
index 0000000..d675e55
--- /dev/null
+++ b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/PEventsSpec.scala
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage._
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.specs2._
+import org.specs2.specification.Step
+
+class PEventsSpec extends Specification with TestEvents {
+
+ System.clearProperty("spark.driver.port")
+ System.clearProperty("spark.hostPort")
+ val sc = new SparkContext("local[4]", "PEventAggregatorSpec test")
+
+ val appId = 1
+ val channelId = 6
+ val dbName = "test_pio_storage_events_" + hashCode
+
+ def hbLocal = Storage.getDataObject[LEvents](
+ StorageTestUtils.hbaseSourceName,
+ dbName
+ )
+
+ def hbPar = Storage.getDataObject[PEvents](
+ StorageTestUtils.hbaseSourceName,
+ dbName
+ )
+
+ def stopSpark = {
+ sc.stop()
+ }
+
+ def is = s2"""
+
+ PredictionIO Storage PEvents Specification
+
+ PEvents can be implemented by:
+ - HBPEvents ${hbPEvents}
+ - (stop Spark) ${Step(sc.stop())}
+
+ """
+
+ def hbPEvents = sequential ^ s2"""
+
+ HBPEvents should
+ - behave like any PEvents implementation ${events(hbLocal, hbPar)}
+ - (table cleanup) ${Step(StorageTestUtils.dropHBaseNamespace(dbName))}
+
+ """
+
+ def events(localEventClient: LEvents, parEventClient: PEvents) = sequential ^ s2"""
+
+ - (init test) ${initTest(localEventClient)}
+ - (insert test events) ${insertTestEvents(localEventClient)}
+ find in default ${find(parEventClient)}
+ find in channel ${findChannel(parEventClient)}
+ aggregate user properties in default ${aggregateUserProperties(parEventClient)}
+ aggregate user properties in channel ${aggregateUserPropertiesChannel(parEventClient)}
+ write to default ${write(parEventClient)}
+ write to channel ${writeChannel(parEventClient)}
+
+ """
+
+ /* setup */
+
+ // events from TestEvents trait
+ val listOfEvents = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2, r1, r2)
+ val listOfEventsChannel = List(u3e1, u3e2, u3e3, r3, r4)
+
+ def initTest(localEventClient: LEvents) = {
+ localEventClient.init(appId)
+ localEventClient.init(appId, Some(channelId))
+ }
+
+ def insertTestEvents(localEventClient: LEvents) = {
+ listOfEvents.map( localEventClient.insert(_, appId) )
+ // insert to channel
+ listOfEventsChannel.map( localEventClient.insert(_, appId, Some(channelId)) )
+ success
+ }
+
+ /* following are tests */
+
+ def find(parEventClient: PEvents) = {
+ val resultRDD: RDD[Event] = parEventClient.find(
+ appId = appId
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map {_.copy(eventId = None)} // ignore eventId
+
+ results must containTheSameElementsAs(listOfEvents)
+ }
+
+ def findChannel(parEventClient: PEvents) = {
+ val resultRDD: RDD[Event] = parEventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map {_.copy(eventId = None)} // ignore eventId
+
+ results must containTheSameElementsAs(listOfEventsChannel)
+ }
+
+ def aggregateUserProperties(parEventClient: PEvents) = {
+ val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
+ appId = appId,
+ entityType = "user"
+ )(sc)
+ val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
+
+ val expected = Map(
+ "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
+ "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateUserPropertiesChannel(parEventClient: PEvents) = {
+ val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
+ appId = appId,
+ channelId = Some(channelId),
+ entityType = "user"
+ )(sc)
+ val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
+
+ val expected = Map(
+ "u3" -> PropertyMap(u3, u3BaseTime, u3LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def write(parEventClient: PEvents) = {
+ val written = List(r5, r6)
+ val writtenRDD = sc.parallelize(written)
+ parEventClient.write(writtenRDD, appId)(sc)
+
+ // read back
+ val resultRDD = parEventClient.find(
+ appId = appId
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map { _.copy(eventId = None)} // ignore eventId
+
+ val expected = listOfEvents ++ written
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def writeChannel(parEventClient: PEvents) = {
+ val written = List(r1, r5, r6)
+ val writtenRDD = sc.parallelize(written)
+ parEventClient.write(writtenRDD, appId, Some(channelId))(sc)
+
+ // read back
+ val resultRDD = parEventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map { _.copy(eventId = None)} // ignore eventId
+
+ val expected = listOfEventsChannel ++ written
+
+ results must containTheSameElementsAs(expected)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/StorageTestUtils.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/StorageTestUtils.scala b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/StorageTestUtils.scala
new file mode 100644
index 0000000..161cf90
--- /dev/null
+++ b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/StorageTestUtils.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage.{LEvents, Storage}
+
+object StorageTestUtils {
+ val hbaseSourceName = "HBASE"
+
+ def dropHBaseNamespace(namespace: String): Unit = {
+ val eventDb = Storage.getDataObject[LEvents](hbaseSourceName, namespace)
+ .asInstanceOf[HBLEvents]
+ val admin = eventDb.client.admin
+ val tableNames = admin.listTableNamesByNamespace(namespace)
+ tableNames.foreach { name =>
+ admin.disableTable(name)
+ admin.deleteTable(name)
+ }
+
+ // Only empty namespaces (no tables) can be removed.
+ admin.deleteNamespace(namespace)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/TestEvents.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/TestEvents.scala b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/TestEvents.scala
new file mode 100644
index 0000000..2171864
--- /dev/null
+++ b/storage/hbase/src/test/scala/org/apache/predictionio/data/storage/hbase/TestEvents.scala
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage.{DataMap, Event}
+import org.joda.time.{DateTime, DateTimeZone}
+
+trait TestEvents {
+
+ val u1BaseTime = new DateTime(654321)
+ val u2BaseTime = new DateTime(6543210)
+ val u3BaseTime = new DateTime(6543410)
+
+ // u1 events
+ val u1e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u1",
+ properties = DataMap(
+ """{
+ "a" : 1,
+ "b" : "value2",
+ "d" : [1, 2, 3],
+ }"""),
+ eventTime = u1BaseTime
+ )
+
+ val u1e2 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"a" : 2}"""),
+ eventTime = u1BaseTime.plusDays(1)
+ )
+
+ val u1e3 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value4"}"""),
+ eventTime = u1BaseTime.plusDays(2)
+ )
+
+ val u1e4 = u1e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"b" : null}"""),
+ eventTime = u1BaseTime.plusDays(3)
+ )
+
+ val u1e5 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"e" : "new"}"""),
+ eventTime = u1BaseTime.plusDays(4)
+ )
+
+ val u1LastTime = u1BaseTime.plusDays(4)
+ val u1 = """{"a": 2, "d": [1, 2, 3], "e": "new"}"""
+
+ // delete event for u1
+ val u1ed = u1e1.copy(
+ event = "$delete",
+ properties = DataMap(),
+ eventTime = u1BaseTime.plusDays(5)
+ )
+
+ // u2 events
+ val u2e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u2",
+ properties = DataMap(
+ """{
+ "a" : 21,
+ "b" : "value12",
+ "d" : [7, 5, 6],
+ }"""),
+ eventTime = u2BaseTime
+ )
+
+ val u2e2 = u2e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"a" : null}"""),
+ eventTime = u2BaseTime.plusDays(1)
+ )
+
+ val u2e3 = u2e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value9", "g": "new11"}"""),
+ eventTime = u2BaseTime.plusDays(2)
+ )
+
+ val u2LastTime = u2BaseTime.plusDays(2)
+ val u2 = """{"b": "value9", "d": [7, 5, 6], "g": "new11"}"""
+
+ // u3 events
+ val u3e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u3",
+ properties = DataMap(
+ """{
+ "a" : 22,
+ "b" : "value13",
+ "d" : [5, 6, 1],
+ }"""),
+ eventTime = u3BaseTime
+ )
+
+ val u3e2 = u3e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"a" : null}"""),
+ eventTime = u3BaseTime.plusDays(1)
+ )
+
+ val u3e3 = u3e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value10", "f": "new12", "d" : [1, 3, 2]}"""),
+ eventTime = u3BaseTime.plusDays(2)
+ )
+
+ val u3LastTime = u3BaseTime.plusDays(2)
+ val u3 = """{"b": "value10", "d": [1, 3, 2], "f": "new12"}"""
+
+ // some random events
+ val r1 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now,
+ prId = Some("my_prid")
+ )
+ val r2 = Event(
+ event = "my_event2",
+ entityType = "my_entity_type2",
+ entityId = "my_entity_id2"
+ )
+ val r3 = Event(
+ event = "my_event3",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "propA" : 1.2345,
+ "propB" : "valueB",
+ }"""
+ ),
+ prId = Some("my_prid")
+ )
+ val r4 = Event(
+ event = "my_event4",
+ entityType = "my_entity_type4",
+ entityId = "my_entity_id4",
+ targetEntityType = Some("my_target_entity_type4"),
+ targetEntityId = Some("my_target_entity_id4"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""),
+ eventTime = DateTime.now
+ )
+ val r5 = Event(
+ event = "my_event5",
+ entityType = "my_entity_type5",
+ entityId = "my_entity_id5",
+ targetEntityType = Some("my_target_entity_type5"),
+ targetEntityId = Some("my_target_entity_id5"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now
+ )
+ val r6 = Event(
+ event = "my_event6",
+ entityType = "my_entity_type6",
+ entityId = "my_entity_id6",
+ targetEntityType = Some("my_target_entity_type6"),
+ targetEntityId = Some("my_target_entity_id6"),
+ properties = DataMap(
+ """{
+ "prop1" : 6,
+ "prop2" : "value2",
+ "prop3" : [6, 7, 8],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now
+ )
+
+ // timezone
+ val tz1 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id0",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = new DateTime(12345678, DateTimeZone.forID("-08:00")),
+ prId = Some("my_prid")
+ )
+
+ val tz2 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id1",
+ eventTime = new DateTime(12345678, DateTimeZone.forID("+02:00")),
+ prId = Some("my_prid")
+ )
+
+ val tz3 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id2",
+ eventTime = new DateTime(12345678, DateTimeZone.forID("+08:00")),
+ prId = Some("my_prid")
+ )
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/LEventsSpec.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/LEventsSpec.scala b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/LEventsSpec.scala
new file mode 100644
index 0000000..d723d07
--- /dev/null
+++ b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/LEventsSpec.scala
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import org.apache.predictionio.data.storage.{Event, LEvents, PropertyMap, Storage}
+import org.specs2._
+import org.specs2.specification.Step
+
+class LEventsSpec extends Specification with TestEvents {
+ def is = s2"""
+
+ PredictionIO Storage LEvents Specification
+
+ Events can be implemented by:
+ - JDBCLEvents ${jdbcLEvents}
+
+ """
+
+ def jdbcLEvents = sequential ^ s2"""
+
+ JDBCLEvents should
+ - behave like any LEvents implementation ${events(jdbcDO)}
+
+ """
+
+ val appId = 1
+
+ def events(eventClient: LEvents) = sequential ^ s2"""
+
+ init default ${initDefault(eventClient)}
+ insert 3 test events and get back by event ID ${insertAndGetEvents(eventClient)}
+ insert 3 test events with timezone and get back by event ID ${insertAndGetTimezone(eventClient)}
+ insert and delete by ID ${insertAndDelete(eventClient)}
+ insert test user events ${insertTestUserEvents(eventClient)}
+ find user events ${findUserEvents(eventClient)}
+ aggregate user properties ${aggregateUserProperties(eventClient)}
+ aggregate one user properties ${aggregateOneUserProperties(eventClient)}
+ aggregate non-existent user properties ${aggregateNonExistentUserProperties(eventClient)}
+ init channel ${initChannel(eventClient)}
+ insert 2 events to channel ${insertChannel(eventClient)}
+ insert 1 event to channel and delete by ID ${insertAndDeleteChannel(eventClient)}
+ find events from channel ${findChannel(eventClient)}
+ remove default ${removeDefault(eventClient)}
+ remove channel ${removeChannel(eventClient)}
+
+ """
+
+ val dbName = "test_pio_storage_events_" + hashCode
+
+ def jdbcDO = Storage.getDataObject[LEvents](StorageTestUtils.jdbcSourceName, dbName)
+
+ def initDefault(eventClient: LEvents) = {
+ eventClient.init(appId)
+ }
+
+ def insertAndGetEvents(eventClient: LEvents) = {
+
+ // events from TestEvents trait
+ val listOfEvents = List(r1,r2,r3)
+
+ val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
+
+ val insertedEventId: List[String] = insertResp
+
+ val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
+ .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
+
+ val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
+
+ val getEvents = getResp
+
+ insertedEvent must containTheSameElementsAs(getEvents)
+ }
+
+ def insertAndGetTimezone(eventClient: LEvents) = {
+ val listOfEvents = List(tz1, tz2, tz3)
+
+ val insertResp = listOfEvents.map { eventClient.insert(_, appId) }
+
+ val insertedEventId: List[String] = insertResp
+
+ val insertedEvent: List[Option[Event]] = listOfEvents.zip(insertedEventId)
+ .map { case (e, id) => Some(e.copy(eventId = Some(id))) }
+
+ val getResp = insertedEventId.map { id => eventClient.get(id, appId) }
+
+ val getEvents = getResp
+
+ insertedEvent must containTheSameElementsAs(getEvents)
+ }
+
+ def insertAndDelete(eventClient: LEvents) = {
+ val eventId = eventClient.insert(r2, appId)
+
+ val resultBefore = eventClient.get(eventId, appId)
+
+ val expectedBefore = r2.copy(eventId = Some(eventId))
+
+ val deleteStatus = eventClient.delete(eventId, appId)
+
+ val resultAfter = eventClient.get(eventId, appId)
+
+ (resultBefore must beEqualTo(Some(expectedBefore))) and
+ (deleteStatus must beEqualTo(true)) and
+ (resultAfter must beEqualTo(None))
+ }
+
+ def insertTestUserEvents(eventClient: LEvents) = {
+ // events from TestEvents trait
+ val listOfEvents = Vector(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
+
+ listOfEvents.map{ eventClient.insert(_, appId) }
+
+ success
+ }
+
+ def findUserEvents(eventClient: LEvents) = {
+
+ val results: List[Event] = eventClient.find(
+ appId = appId,
+ entityType = Some("user"))
+ .toList
+ .map(e => e.copy(eventId = None)) // ignore eventID
+
+ // same events in insertTestUserEvents
+ val expected = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2)
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def aggregateUserProperties(eventClient: LEvents) = {
+
+ val result: Map[String, PropertyMap] = eventClient.aggregateProperties(
+ appId = appId,
+ entityType = "user")
+
+ val expected = Map(
+ "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
+ "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateOneUserProperties(eventClient: LEvents) = {
+ val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
+ appId = appId,
+ entityType = "user",
+ entityId = "u1")
+
+ val expected = Some(PropertyMap(u1, u1BaseTime, u1LastTime))
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateNonExistentUserProperties(eventClient: LEvents) = {
+ val result: Option[PropertyMap] = eventClient.aggregatePropertiesOfEntity(
+ appId = appId,
+ entityType = "user",
+ entityId = "u999999")
+
+ result must beEqualTo(None)
+ }
+
+ val channelId = 12
+
+ def initChannel(eventClient: LEvents) = {
+ eventClient.init(appId, Some(channelId))
+ }
+
+ def insertChannel(eventClient: LEvents) = {
+
+ // events from TestEvents trait
+ val listOfEvents = List(r4,r5)
+
+ listOfEvents.map( eventClient.insert(_, appId, Some(channelId)) )
+
+ success
+ }
+
+ def insertAndDeleteChannel(eventClient: LEvents) = {
+
+ val eventId = eventClient.insert(r2, appId, Some(channelId))
+
+ val resultBefore = eventClient.get(eventId, appId, Some(channelId))
+
+ val expectedBefore = r2.copy(eventId = Some(eventId))
+
+ val deleteStatus = eventClient.delete(eventId, appId, Some(channelId))
+
+ val resultAfter = eventClient.get(eventId, appId, Some(channelId))
+
+ (resultBefore must beEqualTo(Some(expectedBefore))) and
+ (deleteStatus must beEqualTo(true)) and
+ (resultAfter must beEqualTo(None))
+ }
+
+ def findChannel(eventClient: LEvents) = {
+
+ val results: List[Event] = eventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )
+ .toList
+ .map(e => e.copy(eventId = None)) // ignore eventId
+
+ // same events in insertChannel
+ val expected = List(r4, r5)
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def removeDefault(eventClient: LEvents) = {
+ eventClient.remove(appId)
+ }
+
+ def removeChannel(eventClient: LEvents) = {
+ eventClient.remove(appId, Some(channelId))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/PEventsSpec.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/PEventsSpec.scala b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/PEventsSpec.scala
new file mode 100644
index 0000000..71ebf5f
--- /dev/null
+++ b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/PEventsSpec.scala
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import org.apache.predictionio.data.storage._
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.specs2._
+import org.specs2.specification.Step
+
+class PEventsSpec extends Specification with TestEvents {
+
+ System.clearProperty("spark.driver.port")
+ System.clearProperty("spark.hostPort")
+ val sc = new SparkContext("local[4]", "PEventAggregatorSpec test")
+
+ val appId = 1
+ val channelId = 6
+ val dbName = "test_pio_storage_events_" + hashCode
+
+ def jdbcLocal = Storage.getDataObject[LEvents](
+ StorageTestUtils.jdbcSourceName,
+ dbName
+ )
+
+ def jdbcPar = Storage.getDataObject[PEvents](
+ StorageTestUtils.jdbcSourceName,
+ dbName
+ )
+
+ def stopSpark = {
+ sc.stop()
+ }
+
+ def is = s2"""
+
+ PredictionIO Storage PEvents Specification
+
+ PEvents can be implemented by:
+ - JDBCPEvents ${jdbcPEvents}
+ - (stop Spark) ${Step(sc.stop())}
+
+ """
+
+ def jdbcPEvents = sequential ^ s2"""
+
+ JDBCPEvents should
+ - behave like any PEvents implementation ${events(jdbcLocal, jdbcPar)}
+ - (table cleanup) ${Step(StorageTestUtils.dropJDBCTable(s"${dbName}_$appId"))}
+ - (table cleanup) ${Step(StorageTestUtils.dropJDBCTable(s"${dbName}_${appId}_$channelId"))}
+
+ """
+
+ def events(localEventClient: LEvents, parEventClient: PEvents) = sequential ^ s2"""
+
+ - (init test) ${initTest(localEventClient)}
+ - (insert test events) ${insertTestEvents(localEventClient)}
+ find in default ${find(parEventClient)}
+ find in channel ${findChannel(parEventClient)}
+ aggregate user properties in default ${aggregateUserProperties(parEventClient)}
+ aggregate user properties in channel ${aggregateUserPropertiesChannel(parEventClient)}
+ write to default ${write(parEventClient)}
+ write to channel ${writeChannel(parEventClient)}
+
+ """
+
+ /* setup */
+
+ // events from TestEvents trait
+ val listOfEvents = List(u1e5, u2e2, u1e3, u1e1, u2e3, u2e1, u1e4, u1e2, r1, r2)
+ val listOfEventsChannel = List(u3e1, u3e2, u3e3, r3, r4)
+
+ def initTest(localEventClient: LEvents) = {
+ localEventClient.init(appId)
+ localEventClient.init(appId, Some(channelId))
+ }
+
+ def insertTestEvents(localEventClient: LEvents) = {
+ listOfEvents.map( localEventClient.insert(_, appId) )
+ // insert to channel
+ listOfEventsChannel.map( localEventClient.insert(_, appId, Some(channelId)) )
+ success
+ }
+
+ /* following are tests */
+
+ def find(parEventClient: PEvents) = {
+ val resultRDD: RDD[Event] = parEventClient.find(
+ appId = appId
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map {_.copy(eventId = None)} // ignore eventId
+
+ results must containTheSameElementsAs(listOfEvents)
+ }
+
+ def findChannel(parEventClient: PEvents) = {
+ val resultRDD: RDD[Event] = parEventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map {_.copy(eventId = None)} // ignore eventId
+
+ results must containTheSameElementsAs(listOfEventsChannel)
+ }
+
+ def aggregateUserProperties(parEventClient: PEvents) = {
+ val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
+ appId = appId,
+ entityType = "user"
+ )(sc)
+ val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
+
+ val expected = Map(
+ "u1" -> PropertyMap(u1, u1BaseTime, u1LastTime),
+ "u2" -> PropertyMap(u2, u2BaseTime, u2LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def aggregateUserPropertiesChannel(parEventClient: PEvents) = {
+ val resultRDD: RDD[(String, PropertyMap)] = parEventClient.aggregateProperties(
+ appId = appId,
+ channelId = Some(channelId),
+ entityType = "user"
+ )(sc)
+ val result: Map[String, PropertyMap] = resultRDD.collectAsMap.toMap
+
+ val expected = Map(
+ "u3" -> PropertyMap(u3, u3BaseTime, u3LastTime)
+ )
+
+ result must beEqualTo(expected)
+ }
+
+ def write(parEventClient: PEvents) = {
+ val written = List(r5, r6)
+ val writtenRDD = sc.parallelize(written)
+ parEventClient.write(writtenRDD, appId)(sc)
+
+ // read back
+ val resultRDD = parEventClient.find(
+ appId = appId
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map { _.copy(eventId = None)} // ignore eventId
+
+ val expected = listOfEvents ++ written
+
+ results must containTheSameElementsAs(expected)
+ }
+
+ def writeChannel(parEventClient: PEvents) = {
+ val written = List(r1, r5, r6)
+ val writtenRDD = sc.parallelize(written)
+ parEventClient.write(writtenRDD, appId, Some(channelId))(sc)
+
+ // read back
+ val resultRDD = parEventClient.find(
+ appId = appId,
+ channelId = Some(channelId)
+ )(sc)
+
+ val results = resultRDD.collect.toList
+ .map { _.copy(eventId = None)} // ignore eventId
+
+ val expected = listOfEventsChannel ++ written
+
+ results must containTheSameElementsAs(expected)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/StorageTestUtils.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/StorageTestUtils.scala b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/StorageTestUtils.scala
new file mode 100644
index 0000000..4bf90cf
--- /dev/null
+++ b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/StorageTestUtils.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import scalikejdbc._
+
+object StorageTestUtils {
+ val jdbcSourceName = "PGSQL"
+
+ def dropJDBCTable(table: String): Unit = DB autoCommit { implicit s =>
+ SQL(s"drop table $table").execute().apply()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/647b480b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/TestEvents.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/TestEvents.scala b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/TestEvents.scala
new file mode 100644
index 0000000..2cb08e5
--- /dev/null
+++ b/storage/jdbc/src/test/scala/org/apache/predictionio/data/storage/jdbc/TestEvents.scala
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import org.apache.predictionio.data.storage.{DataMap, Event}
+import org.joda.time.{DateTime, DateTimeZone}
+
+trait TestEvents {
+
+ val u1BaseTime = new DateTime(654321)
+ val u2BaseTime = new DateTime(6543210)
+ val u3BaseTime = new DateTime(6543410)
+
+ // u1 events
+ val u1e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u1",
+ properties = DataMap(
+ """{
+ "a" : 1,
+ "b" : "value2",
+ "d" : [1, 2, 3],
+ }"""),
+ eventTime = u1BaseTime
+ )
+
+ val u1e2 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"a" : 2}"""),
+ eventTime = u1BaseTime.plusDays(1)
+ )
+
+ val u1e3 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value4"}"""),
+ eventTime = u1BaseTime.plusDays(2)
+ )
+
+ val u1e4 = u1e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"b" : null}"""),
+ eventTime = u1BaseTime.plusDays(3)
+ )
+
+ val u1e5 = u1e1.copy(
+ event = "$set",
+ properties = DataMap("""{"e" : "new"}"""),
+ eventTime = u1BaseTime.plusDays(4)
+ )
+
+ val u1LastTime = u1BaseTime.plusDays(4)
+ val u1 = """{"a": 2, "d": [1, 2, 3], "e": "new"}"""
+
+ // delete event for u1
+ val u1ed = u1e1.copy(
+ event = "$delete",
+ properties = DataMap(),
+ eventTime = u1BaseTime.plusDays(5)
+ )
+
+ // u2 events
+ val u2e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u2",
+ properties = DataMap(
+ """{
+ "a" : 21,
+ "b" : "value12",
+ "d" : [7, 5, 6],
+ }"""),
+ eventTime = u2BaseTime
+ )
+
+ val u2e2 = u2e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"a" : null}"""),
+ eventTime = u2BaseTime.plusDays(1)
+ )
+
+ val u2e3 = u2e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value9", "g": "new11"}"""),
+ eventTime = u2BaseTime.plusDays(2)
+ )
+
+ val u2LastTime = u2BaseTime.plusDays(2)
+ val u2 = """{"b": "value9", "d": [7, 5, 6], "g": "new11"}"""
+
+ // u3 events
+ val u3e1 = Event(
+ event = "$set",
+ entityType = "user",
+ entityId = "u3",
+ properties = DataMap(
+ """{
+ "a" : 22,
+ "b" : "value13",
+ "d" : [5, 6, 1],
+ }"""),
+ eventTime = u3BaseTime
+ )
+
+ val u3e2 = u3e1.copy(
+ event = "$unset",
+ properties = DataMap("""{"a" : null}"""),
+ eventTime = u3BaseTime.plusDays(1)
+ )
+
+ val u3e3 = u3e1.copy(
+ event = "$set",
+ properties = DataMap("""{"b" : "value10", "f": "new12", "d" : [1, 3, 2]}"""),
+ eventTime = u3BaseTime.plusDays(2)
+ )
+
+ val u3LastTime = u3BaseTime.plusDays(2)
+ val u3 = """{"b": "value10", "d": [1, 3, 2], "f": "new12"}"""
+
+ // some random events
+ val r1 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now,
+ prId = Some("my_prid")
+ )
+ val r2 = Event(
+ event = "my_event2",
+ entityType = "my_entity_type2",
+ entityId = "my_entity_id2"
+ )
+ val r3 = Event(
+ event = "my_event3",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "propA" : 1.2345,
+ "propB" : "valueB",
+ }"""
+ ),
+ prId = Some("my_prid")
+ )
+ val r4 = Event(
+ event = "my_event4",
+ entityType = "my_entity_type4",
+ entityId = "my_entity_id4",
+ targetEntityType = Some("my_target_entity_type4"),
+ targetEntityId = Some("my_target_entity_id4"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""),
+ eventTime = DateTime.now
+ )
+ val r5 = Event(
+ event = "my_event5",
+ entityType = "my_entity_type5",
+ entityId = "my_entity_id5",
+ targetEntityType = Some("my_target_entity_type5"),
+ targetEntityId = Some("my_target_entity_id5"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now
+ )
+ val r6 = Event(
+ event = "my_event6",
+ entityType = "my_entity_type6",
+ entityId = "my_entity_id6",
+ targetEntityType = Some("my_target_entity_type6"),
+ targetEntityId = Some("my_target_entity_id6"),
+ properties = DataMap(
+ """{
+ "prop1" : 6,
+ "prop2" : "value2",
+ "prop3" : [6, 7, 8],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = DateTime.now
+ )
+
+ // timezone
+ val tz1 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id0",
+ targetEntityType = Some("my_target_entity_type"),
+ targetEntityId = Some("my_target_entity_id"),
+ properties = DataMap(
+ """{
+ "prop1" : 1,
+ "prop2" : "value2",
+ "prop3" : [1, 2, 3],
+ "prop4" : true,
+ "prop5" : ["a", "b", "c"],
+ "prop6" : 4.56
+ }"""
+ ),
+ eventTime = new DateTime(12345678, DateTimeZone.forID("-08:00")),
+ prId = Some("my_prid")
+ )
+
+ val tz2 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id1",
+ eventTime = new DateTime(12345678, DateTimeZone.forID("+02:00")),
+ prId = Some("my_prid")
+ )
+
+ val tz3 = Event(
+ event = "my_event",
+ entityType = "my_entity_type",
+ entityId = "my_entity_id2",
+ eventTime = new DateTime(12345678, DateTimeZone.forID("+08:00")),
+ prId = Some("my_prid")
+ )
+
+}
[18/50] [abbrv] incubator-predictionio git commit: Fix output paths
Posted by do...@apache.org.
Fix output paths
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/57aa82f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/57aa82f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/57aa82f1
Branch: refs/heads/livedoc
Commit: 57aa82f1fc8083831d3dda24444a22246dc57372
Parents: 2d388db
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 9 08:55:23 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 9 08:55:23 2017 -0800
----------------------------------------------------------------------
storage/elasticsearch/build.sbt | 2 +-
storage/elasticsearch1/build.sbt | 3 +--
storage/hbase/build.sbt | 3 +--
storage/hdfs/build.sbt | 3 +--
storage/jdbc/build.sbt | 3 +--
storage/localfs/build.sbt | 3 +--
6 files changed, 6 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 925b66f..7d9b2ad 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -53,4 +53,4 @@ assemblyShadeRules in assembly := Seq(
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index 457aba2..dde7285 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -43,5 +43,4 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
-
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 673697b..513e294 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -52,5 +52,4 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
-
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 35c9153..7ddc86c 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -40,5 +40,4 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
-
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 0a07ee0..2011722 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -43,5 +43,4 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
-
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/57aa82f1/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index f94b355..f9e101d 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -40,5 +40,4 @@ assemblyMergeStrategy in assembly := {
// skip test in assembly
test in assembly := {}
-assemblyOutputPath := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
-
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
[26/50] [abbrv] incubator-predictionio git commit: Update event
serialization for ES5
Posted by do...@apache.org.
Update event serialization for ES5
Closes #358
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/ae51040b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/ae51040b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/ae51040b
Branch: refs/heads/livedoc
Commit: ae51040baaebd30608198841b7d4caee2c4c5cd0
Parents: 4639393
Author: Shinsuke Sugaya <sh...@yahoo.co.jp>
Authored: Tue Mar 14 15:08:33 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 14 15:08:33 2017 -0700
----------------------------------------------------------------------
.../storage/elasticsearch/ESEventsUtil.scala | 44 ++------
.../data/storage/elasticsearch/ESLEvents.scala | 35 ++++---
.../data/storage/elasticsearch/ESUtils.scala | 103 +++++++++++++++++--
3 files changed, 121 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ae51040b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
index 56f47ab..2edbc35 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEventsUtil.scala
@@ -18,16 +18,14 @@
package org.apache.predictionio.data.storage.elasticsearch
-import org.apache.hadoop.io.DoubleWritable
-import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.MapWritable
import org.apache.hadoop.io.Text
import org.apache.predictionio.data.storage.DataMap
import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventValidation
import org.joda.time.DateTime
-import org.joda.time.DateTimeZone
import org.json4s._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
object ESEventsUtil {
@@ -53,23 +51,8 @@ object ESEventsUtil {
}
}
- val tmp = result
- .get(new Text("properties")).asInstanceOf[MapWritable]
- .get(new Text("fields")).asInstanceOf[MapWritable]
- .get(new Text("rating"))
-
- val rating =
- if (tmp.isInstanceOf[DoubleWritable]) tmp.asInstanceOf[DoubleWritable]
- else if (tmp.isInstanceOf[LongWritable]) {
- new DoubleWritable(tmp.asInstanceOf[LongWritable].get().toDouble)
- }
- else null
-
- val properties: DataMap =
- if (rating != null) DataMap(s"""{"rating":${rating.get().toString}}""")
- else DataMap()
-
-
+ val properties: DataMap = getOptStringCol("properties")
+ .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
val eventId = Some(getStringCol("eventId"))
val event = getStringCol("event")
val entityType = getStringCol("entityType")
@@ -77,17 +60,8 @@ object ESEventsUtil {
val targetEntityType = getOptStringCol("targetEntityType")
val targetEntityId = getOptStringCol("targetEntityId")
val prId = getOptStringCol("prId")
- val eventTimeZone = getOptStringCol("eventTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
- val eventTime = new DateTime(
- getStringCol("eventTime"), eventTimeZone)
- val creationTimeZone = getOptStringCol("creationTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
- val creationTime: DateTime = new DateTime(
- getStringCol("creationTime"), creationTimeZone)
-
+ val eventTime: DateTime = ESUtils.parseUTCDateTime(getStringCol("eventTime"))
+ val creationTime: DateTime = ESUtils.parseUTCDateTime(getStringCol("creationTime"))
Event(
eventId = eventId,
@@ -112,11 +86,11 @@ object ESEventsUtil {
"entityId" -> event.entityId,
"targetEntityType" -> event.targetEntityType,
"targetEntityId" -> event.targetEntityId,
- "properties" -> event.properties.toJObject,
- "eventTime" -> event.eventTime.toString,
+ "properties" -> write(event.properties.toJObject),
+ "eventTime" -> ESUtils.formatUTCDateTime(event.eventTime),
"tags" -> event.tags,
"prId" -> event.prId,
- "creationTime" -> event.creationTime.toString
+ "creationTime" -> ESUtils.formatUTCDateTime(event.creationTime)
)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ae51040b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
index fdd370a..809a064 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala
@@ -72,17 +72,11 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
("entityId" -> ("type" -> "keyword")) ~
("targetEntityType" -> ("type" -> "keyword")) ~
("targetEntityId" -> ("type" -> "keyword")) ~
- ("properties" ->
- ("type" -> "nested") ~
- ("properties" ->
- ("fields" -> ("type" -> "nested") ~
- ("properties" ->
- ("user" -> ("type" -> "long")) ~
- ("num" -> ("type" -> "long")))))) ~
- ("eventTime" -> ("type" -> "date")) ~
- ("tags" -> ("type" -> "keyword")) ~
- ("prId" -> ("type" -> "keyword")) ~
- ("creationTime" -> ("type" -> "date"))))
+ ("properties" -> ("type" -> "keyword")) ~
+ ("eventTime" -> ("type" -> "date")) ~
+ ("tags" -> ("type" -> "keyword")) ~
+ ("prId" -> ("type" -> "keyword")) ~
+ ("creationTime" -> ("type" -> "date"))))
ESUtils.createMapping(restClient, index, estype, compact(render(json)))
} finally {
restClient.close()
@@ -134,8 +128,19 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
while (exists(restClient, estype, roll)) roll = seq.genNext(seqName)
roll.toString
}
- val json = write(event.copy(eventId = Some(id)))
- val entity = new NStringEntity(json, ContentType.APPLICATION_JSON);
+ val json =
+ ("eventId" -> id) ~
+ ("event" -> event.event) ~
+ ("entityType" -> event.entityType) ~
+ ("entityId" -> event.entityId) ~
+ ("targetEntityType" -> event.targetEntityType) ~
+ ("targetEntityId" -> event.targetEntityId) ~
+ ("eventTime" -> ESUtils.formatUTCDateTime(event.eventTime)) ~
+ ("tags" -> event.tags) ~
+ ("prId" -> event.prId) ~
+ ("creationTime" -> ESUtils.formatUTCDateTime(event.creationTime)) ~
+ ("properties" -> write(event.properties.toJObject))
+ val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON);
val response = restClient.performRequest(
"POST",
s"/$index/$estype/$id",
@@ -275,8 +280,8 @@ class ESLEvents(val client: ESClient, config: StorageClientConfig, val index: St
startTime, untilTime, entityType, entityId,
eventNames, targetEntityType, targetEntityId, reversed)
limit.getOrElse(20) match {
- case -1 => ESUtils.getAll[Event](restClient, index, estype, query).toIterator
- case size => ESUtils.get[Event](restClient, index, estype, query, size).toIterator
+ case -1 => ESUtils.getEventAll(restClient, index, estype, query).toIterator
+ case size => ESUtils.getEvents(restClient, index, estype, query, size).toIterator
}
} catch {
case e: IOException =>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/ae51040b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
index 72f4dd6..4eb117e 100644
--- a/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
+++ b/storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
@@ -34,17 +34,72 @@ import org.joda.time.format.DateTimeFormat
import org.joda.time.DateTimeZone
import org.apache.predictionio.data.storage.StorageClientConfig
import org.apache.http.HttpHost
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.DataMap
object ESUtils {
val scrollLife = "1m"
- def get[T: Manifest](
+ def toEvent(value: JValue)(
+ implicit formats: Formats): Event = {
+ def getString(s: String): String = {
+ (value \ s) match {
+ case x if x == JNothing => null
+ case x => x.extract[String]
+ }
+ }
+
+ def getOptString(s: String): Option[String] = {
+ getString(s) match {
+ case null => None
+ case x => Some(x)
+ }
+ }
+
+ val properties: DataMap = getOptString("properties")
+ .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
+ val eventId = getOptString("eventId")
+ val event = getString("event")
+ val entityType = getString("entityType")
+ val entityId = getString("entityId")
+ val targetEntityType = getOptString("targetEntityType")
+ val targetEntityId = getOptString("targetEntityId")
+ val prId = getOptString("prId")
+ val eventTime: DateTime = ESUtils.parseUTCDateTime(getString("eventTime"))
+ val creationTime: DateTime = ESUtils.parseUTCDateTime(getString("creationTime"))
+ val tags = (value \ "tags").extract[Seq[String]]
+
+ Event(
+ eventId = eventId,
+ event = event,
+ entityType = entityType,
+ entityId = entityId,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ properties = properties,
+ eventTime = eventTime,
+ tags = tags,
+ prId = prId,
+ creationTime = creationTime)
+ }
+
+ def getEvents(
client: RestClient,
index: String,
estype: String,
query: String,
size: Int)(
- implicit formats: Formats): Seq[T] = {
+ implicit formats: Formats): Seq[Event] = {
+ getDocList(client, index, estype, query, size).map(x => toEvent(x))
+ }
+
+ def getDocList(
+ client: RestClient,
+ index: String,
+ estype: String,
+ query: String,
+ size: Int)(
+ implicit formats: Formats): Seq[JValue] = {
val response = client.performRequest(
"POST",
s"/$index/$estype/_search",
@@ -52,7 +107,7 @@ object ESUtils {
new StringEntity(query))
val responseJValue = parse(EntityUtils.toString(response.getEntity))
val hits = (responseJValue \ "hits" \ "hits").extract[Seq[JValue]]
- hits.map(h => (h \ "_source").extract[T])
+ hits.map(h => (h \ "_source"))
}
def getAll[T: Manifest](
@@ -61,9 +116,27 @@ object ESUtils {
estype: String,
query: String)(
implicit formats: Formats): Seq[T] = {
+ getDocAll(client, index, estype, query).map(x => x.extract[T])
+ }
+
+ def getEventAll(
+ client: RestClient,
+ index: String,
+ estype: String,
+ query: String)(
+ implicit formats: Formats): Seq[Event] = {
+ getDocAll(client, index, estype, query).map(x => toEvent(x))
+ }
+
+ def getDocAll(
+ client: RestClient,
+ index: String,
+ estype: String,
+ query: String)(
+ implicit formats: Formats): Seq[JValue] = {
@scala.annotation.tailrec
- def scroll(scrollId: String, hits: Seq[JValue], results: Seq[T]): Seq[T] = {
+ def scroll(scrollId: String, hits: Seq[JValue], results: Seq[JValue]): Seq[JValue] = {
if (hits.isEmpty) results
else {
val json = ("scroll" -> scrollLife) ~ ("scroll_id" -> scrollId)
@@ -76,7 +149,7 @@ object ESUtils {
val responseJValue = parse(EntityUtils.toString(response.getEntity))
scroll((responseJValue \ "_scroll_id").extract[String],
(responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
- hits.map(h => (h \ "_source").extract[T]) ++ results)
+ hits.map(h => (h \ "_source").extract[JValue]) ++ results)
}
}
@@ -87,8 +160,8 @@ object ESUtils {
new StringEntity(query))
val responseJValue = parse(EntityUtils.toString(response.getEntity))
scroll((responseJValue \ "_scroll_id").extract[String],
- (responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
- Nil)
+ (responseJValue \ "hits" \ "hits").extract[Seq[JValue]],
+ Nil)
}
def createIndex(
@@ -131,6 +204,16 @@ object ESUtils {
}
}
+ def formatUTCDateTime(dt: DateTime): String = {
+ DateTimeFormat
+ .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").print(dt.withZone(DateTimeZone.UTC))
+ }
+
+ def parseUTCDateTime(str: String): DateTime = {
+ DateTimeFormat
+ .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").parseDateTime(str)
+ }
+
def createEventQuery(
startTime: Option[DateTime] = None,
untilTime: Option[DateTime] = None,
@@ -142,13 +225,11 @@ object ESUtils {
reversed: Option[Boolean] = None): String = {
val mustQueries = Seq(
startTime.map(x => {
- val v = DateTimeFormat
- .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").print(x.withZone(DateTimeZone.UTC))
+ val v = formatUTCDateTime(x)
s"""{"range":{"eventTime":{"gte":"${v}"}}}"""
}),
untilTime.map(x => {
- val v = DateTimeFormat
- .forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").print(x.withZone(DateTimeZone.UTC))
+ val v = formatUTCDateTime(x)
s"""{"range":{"eventTime":{"lt":"${v}"}}}"""
}),
entityType.map(x => s"""{"term":{"entityType":"${x}"}}"""),
[29/50] [abbrv] incubator-predictionio git commit: Add missing license
Posted by do...@apache.org.
Add missing license
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/bb003243
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/bb003243
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/bb003243
Branch: refs/heads/livedoc
Commit: bb003243835ed20b6471af165303b9da17b09abf
Parents: 647b480
Author: Donald Szeto <do...@apache.org>
Authored: Wed Mar 15 11:31:28 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Wed Mar 15 11:31:28 2017 -0700
----------------------------------------------------------------------
docs/manual/source/templates/index.html.md | 25 +++++++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bb003243/docs/manual/source/templates/index.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/templates/index.html.md b/docs/manual/source/templates/index.html.md
index 41b806b..f60699e 100644
--- a/docs/manual/source/templates/index.html.md
+++ b/docs/manual/source/templates/index.html.md
@@ -2,10 +2,27 @@
title: PredictionIO Official Templates
---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
-PredictionIO's [template gallery](/gallery/template-gallery) offers both official and community contributed Engine Templates for all kinds of machine learning tasks.
-You can easily create one or more engines from these templates .
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+PredictionIO's [template gallery](/gallery/template-gallery) offers both
+official and community contributed Engine Templates for all kinds of machine
+learning tasks. You can easily create one or more engines from these templates.
The section contains more detailed documentation of the official templates for
-beginner users to get familiar with PredictionIO and examples of how the templates
-can be modified for other usages.
+beginner users to get familiar with PredictionIO and examples of how the
+templates can be modified for other usages.
[40/50] [abbrv] incubator-predictionio git commit: [PIO-57] Add SBT
Native Packager
Posted by do...@apache.org.
[PIO-57] Add SBT Native Packager
Closes #359
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d0d2570b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d0d2570b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d0d2570b
Branch: refs/heads/livedoc
Commit: d0d2570b3ec602d1abaad544b5759c016432b733
Parents: dfb01e3
Author: Shinsuke Sugaya <sh...@yahoo.co.jp>
Authored: Thu Mar 30 15:11:01 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 30 15:11:01 2017 -0700
----------------------------------------------------------------------
.gitignore | 7 +-
assembly/build.sbt | 93 ++++++++++++++++++
assembly/src/debian/DEBIAN/postrm | 43 +++++++++
assembly/src/debian/DEBIAN/preinst | 36 +++++++
assembly/src/rpm/scriptlets/postun | 41 ++++++++
assembly/src/rpm/scriptlets/preinst | 35 +++++++
bin/compute-classpath.sh | 2 +-
bin/pio | 44 +++++----
bin/pio-class | 17 +++-
bin/pio-daemon | 35 ++++++-
bin/pio-shell | 34 ++++++-
bin/pio-start-all | 99 +++++++++++---------
bin/pio-stop-all | 50 ++++++----
build.sbt | 3 +
conf/log4j.properties | 2 +-
make-distribution.sh | 22 ++++-
project/plugins.sbt | 2 +
storage/elasticsearch/build.sbt | 3 +-
storage/elasticsearch1/build.sbt | 3 +-
storage/hbase/build.sbt | 3 +-
storage/hdfs/build.sbt | 3 +-
storage/jdbc/build.sbt | 3 +-
storage/localfs/build.sbt | 3 +-
tests/build_docker.sh | 3 +-
tools/build.sbt | 5 +-
.../org/apache/predictionio/tools/Runner.scala | 2 +
.../predictionio/tools/commands/Engine.scala | 2 +
27 files changed, 496 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 68ad5c4..d6dfe21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,7 +12,6 @@ core/data
examples/data/ml-*
fs/
supervisord.conf
-assembly/
/dist
pio.log
*.tar.gz
@@ -20,6 +19,10 @@ pio.log
# Ignore source files whose name prefixed with "Private"
Private*.scala
quickstartapp/
+# Eclipse
+.project
+.classpath
+.settings/
# IntelliJ
*.iml
.idea/
@@ -31,3 +34,5 @@ apache-rat-0.11.jar
tests/dist
tests/docker-files/*.jar
tests/docker-files/*.tgz
+assembly/*.jar
+assembly/src/universal/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/assembly/build.sbt
----------------------------------------------------------------------
diff --git a/assembly/build.sbt b/assembly/build.sbt
new file mode 100644
index 0000000..79fc3ab
--- /dev/null
+++ b/assembly/build.sbt
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import NativePackagerHelper._
+import RpmConstants._
+import com.typesafe.sbt.packager.linux.LinuxSymlink
+
+enablePlugins(RpmPlugin, DebianPlugin)
+
+name := "predictionio"
+
+maintainer in Linux := "Apache Software Foundation"
+packageSummary in Linux := "Apache PredictionIO"
+packageDescription := "Apache PredictionIO is an open source Machine Learning Server " +
+ "built on top of state-of-the-art open source stack for developers " +
+ "and data scientists create predictive engines for any machine learning task."
+
+version in Rpm := version.value.replace("-", "_")
+rpmRelease := "1"
+rpmVendor := "apache"
+rpmGroup := Some("Applications/System")
+rpmUrl := Some("http://predictionio.incubator.apache.org/")
+rpmLicense := Some("Apache License Version 2.0")
+
+maintainerScripts in Rpm := maintainerScriptsAppendFromFile((maintainerScripts in Rpm).value)(
+ Pre -> (sourceDirectory.value / "rpm" / "scriptlets" / "preinst"),
+ Postun -> (sourceDirectory.value / "rpm" / "scriptlets" / "postun")
+)
+
+mappings in Universal ++= {
+ val releaseFile = baseDirectory.value / ".." / "RELEASE.md"
+ val buildPropFile = baseDirectory.value / ".." / "project" / "build.properties"
+ val sbtFile = baseDirectory.value / ".." / "sbt" / "sbt"
+ Seq(releaseFile -> "RELEASE",
+ buildPropFile -> "project/build.properties",
+ sbtFile -> "sbt/sbt")
+}
+
+mappings in Universal ++= {
+ val files = IO.listFiles(baseDirectory.value / ".." / "conf")
+ files filterNot { f => f.getName.endsWith(".travis") } map {
+ case f if f.getName equals "pio-env.sh.template" => f -> "conf/pio-env.sh"
+ case f => f -> s"conf/${f.getName}"
+ } toSeq
+}
+
+mappings in Universal ++= {
+ val files = IO.listFiles(baseDirectory.value / ".." / "bin")
+ files map { f => f -> s"bin/${f.getName}" } toSeq
+}
+
+linuxPackageMappings := {
+ val mappings = linuxPackageMappings.value
+ mappings map { linuxPackage =>
+ val linuxFileMappings = linuxPackage.mappings map {
+ case (f, n) if f.getName equals "conf" => f -> s"/etc/${name.value}"
+ case (f, n) if f.getName equals "pio-env.sh.template" => f -> s"/etc/${name.value}/pio-env.sh"
+ case (f, n) if f.getParent endsWith "conf" => f -> s"/etc/${name.value}/${f.getName}"
+ case (f, n) if f.getName equals "log" => f -> s"/var/log/${name.value}"
+ case (f, n) if f.getName equals "pio.log" => f -> s"/var/log/${name.value}/pio.log"
+ case (f, n) => f -> n
+ }
+
+ val fileData = linuxPackage.fileData.copy(
+ user = s"${name.value}",
+ group = s"${name.value}"
+ )
+
+ linuxPackage.copy(
+ mappings = linuxFileMappings,
+ fileData = fileData
+ )
+ }
+}
+
+linuxPackageSymlinks := {
+ Seq(LinuxSymlink("/usr/bin/pio", s"/usr/share/${name.value}/bin/pio"),
+ LinuxSymlink("/usr/bin/pio-daemon", s"/usr/share/${name.value}/bin/pio-daemon"))
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/assembly/src/debian/DEBIAN/postrm
----------------------------------------------------------------------
diff --git a/assembly/src/debian/DEBIAN/postrm b/assembly/src/debian/DEBIAN/postrm
new file mode 100644
index 0000000..ed9d71c
--- /dev/null
+++ b/assembly/src/debian/DEBIAN/postrm
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+REMOVE_USER_AND_GROUP=false
+
+case "$1" in
+ remove)
+ ;;
+ purge)
+ REMOVE_USER_AND_GROUP=true
+ ;;
+ failed-upgrade|abort-install|abort-upgrade|disappear|upgrade|disappear)
+ ;;
+ *)
+ echo "post remove script called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then
+ if id "predictionio" > /dev/null 2>&1 ; then
+ userdel "predictionio"
+ fi
+
+ if getent group "predictionio" > /dev/null 2>&1 ; then
+ groupdel "predictionio"
+ fi
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/assembly/src/debian/DEBIAN/preinst
----------------------------------------------------------------------
diff --git a/assembly/src/debian/DEBIAN/preinst b/assembly/src/debian/DEBIAN/preinst
new file mode 100644
index 0000000..da60ce3
--- /dev/null
+++ b/assembly/src/debian/DEBIAN/preinst
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if ! getent group "predictionio" > /dev/null 2>&1 ; then
+ echo -n "Creating predictionio group..."
+ addgroup --quiet --system "predictionio"
+ echo " OK"
+fi
+
+if ! id predictionio > /dev/null 2>&1 ; then
+ echo -n "Creating predictionio user..."
+ adduser --quiet \
+ --system \
+ --no-create-home \
+ --ingroup "predictionio" \
+ --disabled-password \
+ --shell /bin/false \
+ --home "/usr/share/predictionio" \
+ "predictionio"
+ echo " OK"
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/assembly/src/rpm/scriptlets/postun
----------------------------------------------------------------------
diff --git a/assembly/src/rpm/scriptlets/postun b/assembly/src/rpm/scriptlets/postun
new file mode 100644
index 0000000..206fb86
--- /dev/null
+++ b/assembly/src/rpm/scriptlets/postun
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+REMOVE_USER_AND_GROUP=false
+
+case "$1" in
+ 0)
+ REMOVE_USER_AND_GROUP=true
+ ;;
+ 1)
+ ;;
+ *)
+ echo "post remove script called with unknown argument \`$1'" >&2
+ exit 1
+ ;;
+esac
+
+if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then
+ if id "predictionio" > /dev/null 2>&1 ; then
+ userdel "predictionio"
+ fi
+
+ if getent group "predictionio" > /dev/null 2>&1 ; then
+ groupdel "predictionio"
+ fi
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/assembly/src/rpm/scriptlets/preinst
----------------------------------------------------------------------
diff --git a/assembly/src/rpm/scriptlets/preinst b/assembly/src/rpm/scriptlets/preinst
new file mode 100644
index 0000000..cb66b46
--- /dev/null
+++ b/assembly/src/rpm/scriptlets/preinst
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if ! getent group "predictionio" > /dev/null 2>&1 ; then
+ echo -n "Creating predictionio group..."
+ groupadd -r "predictionio"
+ echo " OK"
+fi
+
+if ! id predictionio > /dev/null 2>&1 ; then
+ echo -n "Creating predictionio user..."
+ useradd --system \
+ -M \
+ --gid "predictionio" \
+ --shell /sbin/nologin \
+ --comment "fess user" \
+ -d "/usr/share/predictionio" \
+ "predictionio"
+ echo " OK"
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 69cbb25..032d111 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -25,7 +25,7 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
. ${FWDIR}/bin/load-pio-env.sh
# Build up classpath
-CLASSPATH="${FWDIR}/conf"
+CLASSPATH="${PIO_CONF_DIR}"
CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio
----------------------------------------------------------------------
diff --git a/bin/pio b/bin/pio
index fca48f6..01e73ea 100755
--- a/bin/pio
+++ b/bin/pio
@@ -32,28 +32,38 @@ search() {
echo ${i}
}
-PIO_FILE=$(readlink -f $0 2>/dev/null)
-if [ $? = 0 ] ; then
- export PIO_HOME="$(cd $(dirname $PIO_FILE)/..; pwd)"
-else
- CURRENT_DIR=`pwd`
- TARGET_FILE="$0"
- cd "$(dirname "$TARGET_FILE")"
- TARGET_FILE=$(basename "$TARGET_FILE")
-
- while [ -L "$TARGET_FILE" ]
- do
- TARGET_FILE=$(readlink "$TARGET_FILE")
+if [ -z $PIO_HOME ] ; then
+ PIO_FILE=$(readlink -f $0 2>/dev/null)
+ if [ $? = 0 ] ; then
+ export PIO_HOME="$(cd $(dirname $PIO_FILE)/..; pwd)"
+ else
+ CURRENT_DIR=`pwd`
+ TARGET_FILE="$0"
cd "$(dirname "$TARGET_FILE")"
TARGET_FILE=$(basename "$TARGET_FILE")
- done
- export PIO_HOME="$(cd $(dirname "$TARGET_FILE")/..; pwd -P)"
- cd "$CURRENT_DIR"
-fi
+ while [ -L "$TARGET_FILE" ]
+ do
+ TARGET_FILE=$(readlink "$TARGET_FILE")
+ cd "$(dirname "$TARGET_FILE")"
+ TARGET_FILE=$(basename "$TARGET_FILE")
+ done
+ export PIO_HOME="$(cd $(dirname "$TARGET_FILE")/..; pwd -P)"
+ cd "$CURRENT_DIR"
+ fi
+fi
-export PIO_CONF_DIR="${PIO_HOME}/conf"
+if [ -z $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="${PIO_HOME}/conf"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="/etc/predictionio"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ echo "PIO_CONF_DIR is not found."
+ exit 1
+ fi
+ fi
+fi
FIRST_SEP=$(search "--" $@)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio-class
----------------------------------------------------------------------
diff --git a/bin/pio-class b/bin/pio-class
index 002da5c..8bde407 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -40,8 +40,8 @@ if [ -z "$1" ]; then
fi
# Warn if log4j.properties is not present
-if [ ! -f "$FWDIR/conf/log4j.properties" ]; then
- echo -e "\033[0;35mWarning: log4j.properties is missing from $FWDIR/conf\033[0m"
+if [ ! -f "$PIO_CONF_DIR/log4j.properties" ]; then
+ echo -e "\033[0;35mWarning: log4j.properties is missing from $PIO_CONF_DIR\033[0m"
fi
# Make sure the Apache Spark version meets the prerequisite if it is a binary
@@ -84,6 +84,19 @@ else
CLASSPATH=${classpath_output}
fi
+if [ -z $PIO_LOG_DIR ] ; then
+ PIO_LOG_DIR=$PIO_HOME/log
+ touch $PIO_LOG_DIR/pio.log > /dev/null 2>&1
+ if [ $? != 0 ] ; then
+ PIO_LOG_DIR=/var/log/predictionio
+ touch $PIO_LOG_DIR/pio.log > /dev/null 2>&1
+ if [ $? != 0 ] ; then
+ PIO_LOG_DIR=$HOME
+ fi
+ fi
+fi
+
export CLASSPATH
+export JAVA_OPTS="$JAVA_OPTS -Dpio.log.dir=$PIO_LOG_DIR"
exec "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio-daemon
----------------------------------------------------------------------
diff --git a/bin/pio-daemon b/bin/pio-daemon
index 62704e1..630b087 100755
--- a/bin/pio-daemon
+++ b/bin/pio-daemon
@@ -22,7 +22,7 @@ search() {
local needle=$1;
shift
for str in $@; do
- if [ "$str" = "$needle" ]; then
+ if [ "${str}" = "$needle" ]; then
echo ${i}
return
else
@@ -32,9 +32,38 @@ search() {
echo ${i}
}
-export PIO_HOME="$(cd `dirname $0`/..; pwd)"
+if [ -z $PIO_HOME ] ; then
+ PIO_FILE=$(readlink -f $0 2>/dev/null)
+ if [ $? = 0 ] ; then
+ export PIO_HOME="$(cd $(dirname $PIO_FILE)/..; pwd)"
+ else
+ CURRENT_DIR=`pwd`
+ TARGET_FILE="$0"
+ cd "$(dirname "$TARGET_FILE")"
+ TARGET_FILE=$(basename "$TARGET_FILE")
-export PIO_CONF_DIR="$PIO_HOME/conf"
+ while [ -L "$TARGET_FILE" ]
+ do
+ TARGET_FILE=$(readlink "$TARGET_FILE")
+ cd "$(dirname "$TARGET_FILE")"
+ TARGET_FILE=$(basename "$TARGET_FILE")
+ done
+
+ export PIO_HOME="$(cd $(dirname "$TARGET_FILE")/..; pwd -P)"
+ cd "$CURRENT_DIR"
+ fi
+fi
+
+if [ -z $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="${PIO_HOME}/conf"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="/etc/predictionio"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ echo "PIO_CONF_DIR is not found."
+ exit 1
+ fi
+ fi
+fi
PIDFILE=$1
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio-shell
----------------------------------------------------------------------
diff --git a/bin/pio-shell b/bin/pio-shell
index bb57ddb..d47008f 100755
--- a/bin/pio-shell
+++ b/bin/pio-shell
@@ -17,7 +17,39 @@
# limitations under the License.
#
-export PIO_HOME="$(cd `dirname $0`/..; pwd)"
+if [ -z $PIO_HOME ] ; then
+ PIO_FILE=$(readlink -f $0 2>/dev/null)
+ if [ $? = 0 ] ; then
+ export PIO_HOME="$(cd $(dirname $PIO_FILE)/..; pwd)"
+ else
+ CURRENT_DIR=`pwd`
+ TARGET_FILE="$0"
+ cd "$(dirname "$TARGET_FILE")"
+ TARGET_FILE=$(basename "$TARGET_FILE")
+
+ while [ -L "$TARGET_FILE" ]
+ do
+ TARGET_FILE=$(readlink "$TARGET_FILE")
+ cd "$(dirname "$TARGET_FILE")"
+ TARGET_FILE=$(basename "$TARGET_FILE")
+ done
+
+ export PIO_HOME="$(cd $(dirname "$TARGET_FILE")/..; pwd -P)"
+ cd "$CURRENT_DIR"
+ fi
+fi
+
+if [ -z $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="${PIO_HOME}/conf"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ export PIO_CONF_DIR="/etc/predictionio"
+ if [ ! -d $PIO_CONF_DIR ] ; then
+ echo "PIO_CONF_DIR is not found."
+ exit 1
+ fi
+ fi
+fi
+
. ${PIO_HOME}/bin/load-pio-env.sh
if [[ "$1" == "--with-spark" ]]
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio-start-all
----------------------------------------------------------------------
diff --git a/bin/pio-start-all b/bin/pio-start-all
index a78b0d2..15ac1a6 100755
--- a/bin/pio-start-all
+++ b/bin/pio-start-all
@@ -25,63 +25,78 @@ export PIO_HOME="$(cd `dirname $0`/..; pwd)"
. ${PIO_HOME}/bin/load-pio-env.sh
+SOURCE_TYPE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
+SOURCE_TYPE=$SOURCE_TYPE$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
+SOURCE_TYPE=$SOURCE_TYPE$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
+
# Elasticsearch
-echo "Starting Elasticsearch..."
-if [ -n "$PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME" ]; then
- if [ -n "$JAVA_HOME" ]; then
- JPS=`$JAVA_HOME/bin/jps`
- else
- JPS=`jps`
+if [ `echo $SOURCE_TYPE | grep -i elasticsearch | wc -l` != 0 ] ; then
+ echo "Starting Elasticsearch..."
+ if [ -n "$PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME" ]; then
+ ELASTICSEARCH_HOME=$PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME
+ elif [ -n "$PIO_STORAGE_SOURCES_ELASTICSEARCH5_HOME" ]; then
+ ELASTICSEARCH_HOME=$PIO_STORAGE_SOURCES_ELASTICSEARCH5_HOME
fi
- if [[ ${JPS} =~ "Elasticsearch" ]]; then
- echo -e "\033[0;31mElasticsearch is already running. Please use pio-stop-all to try stopping it first.\033[0m"
- echo -e "\033[0;31mNote: If you started Elasticsearch manually, you will need to kill it manually.\033[0m"
- echo -e "\033[0;31mAborting...\033[0m"
- exit 1
+ if [ -n "$ELASTICSEARCH_HOME" ]; then
+ if [ -n "$JAVA_HOME" ]; then
+ JPS=`$JAVA_HOME/bin/jps`
+ else
+ JPS=`jps`
+ fi
+ if [[ ${JPS} =~ "Elasticsearch" ]]; then
+ echo -e "\033[0;31mElasticsearch is already running. Please use pio-stop-all to try stopping it first.\033[0m"
+ echo -e "\033[0;31mNote: If you started Elasticsearch manually, you will need to kill it manually.\033[0m"
+ echo -e "\033[0;31mAborting...\033[0m"
+ exit 1
+ else
+ $ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid
+ fi
else
- $PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME/bin/elasticsearch -d -p $PIO_HOME/es.pid
+ echo -e "\033[0;31mPlease set PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME in conf/pio-env.sh, or in your environment.\033[0m"
+ echo -e "\033[0;31mCannot start Elasticsearch. Aborting...\033[0m"
+ exit 1
fi
-else
- echo -e "\033[0;31mPlease set PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME in conf/pio-env.sh, or in your environment.\033[0m"
- echo -e "\033[0;31mCannot start Elasticsearch. Aborting...\033[0m"
- exit 1
fi
# HBase
-echo "Starting HBase..."
-if [ -n "$PIO_STORAGE_SOURCES_HBASE_HOME" ]; then
- $PIO_STORAGE_SOURCES_HBASE_HOME/bin/start-hbase.sh
-else
- echo -e "\033[0;31mPlease set PIO_STORAGE_SOURCES_HBASE_HOME in conf/pio-env.sh, or in your environment.\033[0m"
- # Kill everything for cleanliness
- echo -e "\033[0;31mCannot start HBase. Aborting...\033[0m"
- sleep 3
- ${PIO_HOME}/bin/pio-stop-all
- exit 1
+if [ `echo $SOURCE_TYPE | grep -i hbase | wc -l` != 0 ] ; then
+ echo "Starting HBase..."
+ if [ -n "$PIO_STORAGE_SOURCES_HBASE_HOME" ]; then
+ $PIO_STORAGE_SOURCES_HBASE_HOME/bin/start-hbase.sh
+ else
+ echo -e "\033[0;31mPlease set PIO_STORAGE_SOURCES_HBASE_HOME in conf/pio-env.sh, or in your environment.\033[0m"
+ # Kill everything for cleanliness
+ echo -e "\033[0;31mCannot start HBase. Aborting...\033[0m"
+ sleep 3
+ ${PIO_HOME}/bin/pio-stop-all
+ exit 1
+ fi
fi
#PGSQL
-pgsqlStatus="$(ps auxwww | grep postgres | wc -l)"
-if [[ "$pgsqlStatus" < 5 ]]; then
- # Detect OS
- OS=`uname`
- if [[ "$OS" = "Darwin" ]]; then
- pg_cmd=`which pg_ctl`
- if [[ "$pg_cmd" != "" ]]; then
- pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
+if [ `echo $SOURCE_TYPE | grep -i pgsql | wc -l` != 0 ] ; then
+ pgsqlStatus="$(ps auxwww | grep postgres | wc -l)"
+ if [[ "$pgsqlStatus" < 5 ]]; then
+ # Detect OS
+ OS=`uname`
+ if [[ "$OS" = "Darwin" ]]; then
+ pg_cmd=`which pg_ctl`
+ if [[ "$pg_cmd" != "" ]]; then
+ pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
+ fi
+ elif [[ "$OS" = "Linux" ]]; then
+ sudo service postgresql start
+ else
+ echo -e "\033[1;31mYour OS $OS is not yet supported for automatic postgresql startup:(\033[0m"
+ echo -e "\033[1;31mPlease do a manual startup!\033[0m"
+ ${PIO_HOME}/bin/pio-stop-all
+ exit 1
fi
- elif [[ "$OS" = "Linux" ]]; then
- sudo service postgresql start
- else
- echo -e "\033[1;31mYour OS $OS is not yet supported for automatic postgresql startup:(\033[0m"
- echo -e "\033[1;31mPlease do a manual startup!\033[0m"
- ${PIO_HOME}/bin/pio-stop-all
- exit 1
fi
fi
# PredictionIO Event Server
-echo "Waiting 10 seconds for HBase to fully initialize..."
+echo "Waiting 10 seconds for Storage Repositories to fully initialize..."
sleep 10
echo "Starting PredictionIO Event Server..."
${PIO_HOME}/bin/pio-daemon ${PIO_HOME}/eventserver.pid eventserver --ip 0.0.0.0
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/bin/pio-stop-all
----------------------------------------------------------------------
diff --git a/bin/pio-stop-all b/bin/pio-stop-all
index 4aab5a3..dabad5d 100755
--- a/bin/pio-stop-all
+++ b/bin/pio-stop-all
@@ -25,6 +25,10 @@ export PIO_HOME="$(cd `dirname $0`/..; pwd)"
. ${PIO_HOME}/bin/load-pio-env.sh
+SOURCE_TYPE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
+SOURCE_TYPE=$SOURCE_TYPE$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
+SOURCE_TYPE=$SOURCE_TYPE$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
+
# PredictionIO Event Server
echo "Stopping PredictionIO Event Server..."
PIDFILE=${PIO_HOME}/eventserver.pid
@@ -34,30 +38,38 @@ if [ -e ${PIDFILE} ]; then
fi
# HBase
-echo "Stopping HBase..."
-if [ -n "$PIO_STORAGE_SOURCES_HBASE_HOME" ]; then
- $PIO_STORAGE_SOURCES_HBASE_HOME/bin/stop-hbase.sh
+if [ `echo $SOURCE_TYPE | grep -i hbase | wc -l` != 0 ] ; then
+ echo "Stopping HBase..."
+ if [ -n "$PIO_STORAGE_SOURCES_HBASE_HOME" ]; then
+ $PIO_STORAGE_SOURCES_HBASE_HOME/bin/stop-hbase.sh
+ fi
fi
# Elasticsearch
-echo "Stopping Elasticsearch..."
-PIDFILE=${PIO_HOME}/es.pid
-if [ -e ${PIDFILE} ]; then
- cat ${PIDFILE} | xargs kill
- rm ${PIDFILE}
+if [ `echo $SOURCE_TYPE | grep -i elasticsearch | wc -l` != 0 ] ; then
+ echo "Stopping Elasticsearch..."
+ PIDFILE=${PIO_HOME}/es.pid
+ if [ -e ${PIDFILE} ]; then
+ cat ${PIDFILE} | xargs kill
+ rm ${PIDFILE}
+ fi
fi
#PGSQL
-OS=`uname`
-if [[ "$OS" = "Darwin" ]]; then
- pg_cmd=`which pg_ctl`
- if [[ "$pg_cmd" != "" ]]; then
- pg_ctl -D /usr/local/var/postgres stop -s -m fast
+if [ `echo $SOURCE_TYPE | grep -i pgsql | wc -l` != 0 ] ; then
+ if [ -n "$PIO_STORAGE_SOURCES_PGSQL_TYPE" ]; then
+ OS=`uname`
+ if [[ "$OS" = "Darwin" ]]; then
+ pg_cmd=`which pg_ctl`
+ if [[ "$pg_cmd" != "" ]]; then
+ pg_ctl -D /usr/local/var/postgres stop -s -m fast
+ fi
+ elif [[ "$OS" = "Linux" ]]; then
+ sudo service postgresql stop
+ else
+ echo -e "\033[1;31mYour OS $OS is not yet supported for automatic postgresql startup:(\033[0m"
+ echo -e "\033[1;31mPlease do a manual shutdown!\033[0m"
+ exit 1
+ fi
fi
-elif [[ "$OS" = "Linux" ]]; then
- sudo service postgresql stop
-else
- echo -e "\033[1;31mYour OS $OS is not yet supported for automatic postgresql startup:(\033[0m"
- echo -e "\033[1;31mPlease do a manual shutdown!\033[0m"
- exit 1
fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 6c4cb7c..06030cb 100644
--- a/build.sbt
+++ b/build.sbt
@@ -185,6 +185,9 @@ val storage = (project in file("storage"))
.aggregate(storageSubprojects map Project.projectToRef: _*)
.disablePlugins(sbtassembly.AssemblyPlugin)
+val assembly = (project in file("assembly")).
+ settings(commonSettings: _*)
+
val root = (project in file(".")).
settings(commonSettings: _*).
enablePlugins(ScalaUnidocPlugin).
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/conf/log4j.properties b/conf/log4j.properties
index a9e9d8e..77a7c60 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -25,7 +25,7 @@ log4j.appender.console.layout.ConversionPattern=[%p] [%c{1}] %m%n%throwable{0}
# file appender
log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.File=./pio.log
+log4j.appender.file.File=${pio.log.dir}/pio.log
log4j.appender.file.layout=org.apache.log4j.EnhancedPatternLayout
log4j.appender.file.layout.ConversionPattern=%d %-5p %c [%t] - %m%n
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index e92178f..03c15e8 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -24,6 +24,9 @@ usage ()
echo "Usage: $0 [-h|--help]"
echo ""
echo " -h|--help Show usage"
+ echo ""
+ echo " --with-rpm Build distribution for RPM package"
+ echo " --with-deb Build distribution for DEB package"
}
JAVA_PROPS=()
@@ -40,6 +43,14 @@ case $i in
JAVA_PROPS+=("$i")
shift
;;
+ --with-rpm)
+ RPM_BUILD=true
+ shift
+ ;;
+ --with-deb)
+ DEB_BUILD=true
+ shift
+ ;;
*)
usage
exit 1
@@ -59,6 +70,13 @@ set -x
sbt/sbt "${JAVA_PROPS[@]}" clean
sbt/sbt "${JAVA_PROPS[@]}" printBuildInfo
sbt/sbt "${JAVA_PROPS[@]}" publishLocal assembly storage/assembly
+sbt/sbt "${JAVA_PROPS[@]}" assembly/clean assembly/universal:packageBin assembly/universal:packageZipTarball
+if [ x$RPM_BUILD = "xtrue" ] ; then
+ sbt/sbt "${JAVA_PROPS[@]}" assembly/rpm:packageBin
+fi
+if [ x$DEB_BUILD = "xtrue" ] ; then
+ sbt/sbt "${JAVA_PROPS[@]}" assembly/debian:packageBin
+fi
set +x
cd ${FWDIR}
@@ -75,8 +93,8 @@ cp ${FWDIR}/bin/* ${DISTDIR}/bin || :
cp ${FWDIR}/conf/* ${DISTDIR}/conf
cp ${FWDIR}/project/build.properties ${DISTDIR}/project
cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
-cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
-cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
+cp ${FWDIR}/assembly/src/universal/lib/*assembly*jar ${DISTDIR}/lib
+cp ${FWDIR}/assembly/src/universal/lib/spark/*jar ${DISTDIR}/lib/spark
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 2f21e00..f13cafd 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -11,3 +11,5 @@ addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/"
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5")
+
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0-M8")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 091b354..50292c5 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -44,4 +44,5 @@ assemblyShadeRules in assembly := Seq(
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / s"pio-data-elasticsearch-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ s"pio-data-elasticsearch-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index 6ed4df0..5e72f91 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -34,4 +34,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / s"pio-data-elasticsearch1-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ s"pio-data-elasticsearch1-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index c33db2d..1e904fa 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -45,4 +45,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / s"pio-data-hbase-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ s"pio-data-hbase-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index f8a2bd5..26a3122 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -35,4 +35,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ ("pio-data-hdfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 3c5026a..c5bcb12 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -36,4 +36,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / s"pio-data-jdbc-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ s"pio-data-jdbc-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index 3413e97..2306d24 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -33,4 +33,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
- "assembly" / "spark" / s"pio-data-localfs-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / "spark" /
+ s"pio-data-localfs-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index 27fb744..d467e97 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -38,8 +38,7 @@ fi
-Dhadoop.version=$PIO_HADOOP_VERSION \
-Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
sbt/sbt clean storage/clean
-rm -rf assembly
-mkdir assembly
+rm -rf assembly/*.jar
cp dist/lib/*.jar assembly/
rm -rf lib/spark
mkdir -p lib/spark
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/tools/build.sbt
----------------------------------------------------------------------
diff --git a/tools/build.sbt b/tools/build.sbt
index 23bb71a..86c4b6d 100644
--- a/tools/build.sbt
+++ b/tools/build.sbt
@@ -53,8 +53,9 @@ assemblyShadeRules in assembly := Seq(
test in assembly := {}
assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile /
- "assembly" / s"pio-assembly-${version.value}.jar"
+ "assembly" / "src" / "universal" / "lib" / s"pio-assembly-${version.value}.jar"
-cleanFiles <+= baseDirectory { base => base.getParentFile / "assembly" }
+cleanFiles <+= baseDirectory { base => base.getParentFile /
+ "assembly" / "src" / "universal" / "lib" }
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index 662dbbf..70e3837 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -189,6 +189,7 @@ object Runner extends EitherLogging {
}
val verboseArg = if (verbose) Seq("--verbose") else Nil
+ val pioLogDir = Option(System.getProperty("pio.log.dir")).getOrElse(s"${pioHome}/log")
val sparkSubmit = Seq(
sparkSubmitCommand,
@@ -198,6 +199,7 @@ object Runner extends EitherLogging {
sparkSubmitFiles,
sparkSubmitExtraClasspaths,
sparkSubmitKryo,
+ Seq("--driver-java-options", s"-Dpio.log.dir=${pioLogDir}"),
Seq(mainJar),
detectFilePaths(fs, sa.scratchUri, classArgs),
Seq("--env", pioEnvVars),
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d0d2570b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index 3924117..e49c3fc 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -294,6 +294,7 @@ object Engine extends EitherLogging {
val jarPluginFiles = jarFilesForSpark(pioHome)
jarPluginFiles foreach { f => info(s"Found JAR: ${f.getName}") }
val allJarFiles = jarFiles.map(_.getCanonicalPath) ++ jarPluginFiles.map(_.getCanonicalPath)
+ val pioLogDir = Option(System.getProperty("pio.log.dir")).getOrElse(s"${pioHome}/log")
val cmd = s"${getSparkHome(sparkArgs.sparkHome)}/bin/spark-submit --jars " +
s"${allJarFiles.mkString(",")} " +
@@ -302,6 +303,7 @@ object Engine extends EitherLogging {
} else {
""
}) +
+ "--driver-java-options -Dpio.log.dir=${pioLogDir} " +
"--class " +
s"${mainClass} ${sparkArgs.sparkPassThrough.mkString(" ")} " +
coreAssembly(pioHome) + " " +
[02/50] [abbrv] incubator-predictionio git commit: Force GitHub sync
Posted by do...@apache.org.
Force GitHub sync
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/f5592226
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/f5592226
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/f5592226
Branch: refs/heads/livedoc
Commit: f5592226f81a637fe123f55b0b5dae142f5a69aa
Parents: f32b941
Author: Donald Szeto <do...@apache.org>
Authored: Sun Feb 26 15:34:24 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Feb 26 15:34:24 2017 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[05/50] [abbrv] incubator-predictionio git commit: Enable event
server to use provided event ID
Posted by do...@apache.org.
Enable event server to use provided event ID
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/43b0e69f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/43b0e69f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/43b0e69f
Branch: refs/heads/livedoc
Commit: 43b0e69f2531c6fd37381e1b96eacc03456a3f40
Parents: 4e3d199
Author: Ahmet DAL <ce...@gmail.com>
Authored: Tue Mar 7 10:21:34 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 10:21:34 2017 -0800
----------------------------------------------------------------------
.../org/apache/predictionio/data/storage/EventJson4sSupport.scala | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/43b0e69f/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala b/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
index 7cfcfc9..ba20c61 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/EventJson4sSupport.scala
@@ -50,6 +50,7 @@ object EventJson4sSupport {
// use getOpt() if not required in json
try {
val event = fields.get[String]("event")
+ val eventId = fields.getOpt[String]("eventId")
val entityType = fields.get[String]("entityType")
val entityId = fields.get[String]("entityId")
val targetEntityType = fields.getOpt[String]("targetEntityType")
@@ -88,6 +89,7 @@ object EventJson4sSupport {
val newEvent = Event(
+ eventId=eventId,
event = event,
entityType = entityType,
entityId = entityId,
[07/50] [abbrv] incubator-predictionio git commit: New engine for
linear regression
Posted by do...@apache.org.
New engine for linear regression
Closes #343
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/aebe43ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/aebe43ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/aebe43ac
Branch: refs/heads/livedoc
Commit: aebe43acfadeb88b98ccc566dbe9056003e1dd85
Parents: b0d37b2
Author: mgcdanny <da...@gmail.com>
Authored: Tue Mar 7 10:27:54 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 10:27:54 2017 -0800
----------------------------------------------------------------------
docs/manual/source/gallery/templates.yaml | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/aebe43ac/docs/manual/source/gallery/templates.yaml
----------------------------------------------------------------------
diff --git a/docs/manual/source/gallery/templates.yaml b/docs/manual/source/gallery/templates.yaml
index 62955f2..8bbaf8b 100644
--- a/docs/manual/source/gallery/templates.yaml
+++ b/docs/manual/source/gallery/templates.yaml
@@ -684,3 +684,20 @@
pio_min_version: 0.9.2
apache_pio_convesion_required: "already compatible"
support_link: '<a href="http://predictionio.incubator.apache.org/support/">Apache PredictionIO mailing lists</a>'
+
+- template:
+ name: Linear Regression BFGS
+ repo: "https://github.com/mgcdanny/pio-linear-regression-bfgs"
+ description: |-
+ Use for:
+
+ * Modeling the relationship between a dependent variable, y, and one or more explanatory variables, denoted X.
+
+ Forked from apache/incubator-predictionio-template-skeleton
+
+ tags: [regression]
+ type: Parallel
+ language: Scala
+ license: "Apache Licence 2.0"
+ status: beta
+ pio_min_version: 0.10.0
[36/50] [abbrv] incubator-predictionio git commit: Remove duplicate
settings
Posted by do...@apache.org.
Remove duplicate settings
Closes #365
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/23a86932
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/23a86932
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/23a86932
Branch: refs/heads/livedoc
Commit: 23a8693282e957fb0bcece08c902184dae9a4e24
Parents: 5f7a59a
Author: shimamoto <ch...@gmail.com>
Authored: Tue Mar 28 08:48:50 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 28 08:48:50 2017 -0700
----------------------------------------------------------------------
build.sbt | 2 --
storage/hdfs/build.sbt | 4 ----
tools/build.sbt | 2 --
3 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/23a86932/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index c0542f7..6c4cb7c 100644
--- a/build.sbt
+++ b/build.sbt
@@ -53,8 +53,6 @@ organization in ThisBuild := "org.apache.predictionio"
scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.10.6")
-crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
-
crossScalaVersions in ThisBuild := Seq(scalaVersion.value, "2.11.8")
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/23a86932/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 3fd8d22..f8a2bd5 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -31,10 +31,6 @@ pomExtra := childrenPomExtra.value
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(
- includeScala = false,
- excludedJars = (fullClasspath in assembly).value.filter {_.data.getName startsWith "apache-predictionio"})
-
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/23a86932/tools/build.sbt
----------------------------------------------------------------------
diff --git a/tools/build.sbt b/tools/build.sbt
index a39c6b0..23bb71a 100644
--- a/tools/build.sbt
+++ b/tools/build.sbt
@@ -27,8 +27,6 @@ libraryDependencies ++= Seq(
"io.spray" %% "spray-testkit" % "1.3.3" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
-dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.18"
-
assemblyMergeStrategy in assembly := {
case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
[45/50] [abbrv] incubator-predictionio git commit: Prepare
0.11.0-incubating-rc1
Posted by do...@apache.org.
Prepare 0.11.0-incubating-rc1
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d5ff55fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d5ff55fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d5ff55fc
Branch: refs/heads/livedoc
Commit: d5ff55fcb468871bf1adfb5500e421890f19fdff
Parents: caa43f8
Author: Donald Szeto <do...@apache.org>
Authored: Mon Apr 3 21:52:11 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Apr 3 21:52:11 2017 -0700
----------------------------------------------------------------------
build.sbt | 2 +-
docs/manual/source/install/install-sourcecode.html.md.erb | 5 ++---
tests/pio_tests/engines/recommendation-engine/build.sbt | 2 +-
3 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d5ff55fc/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index a0a41dd..208bde7 100644
--- a/build.sbt
+++ b/build.sbt
@@ -47,7 +47,7 @@ lazy val scalaSparkDepsVersion = Map(
name := "apache-predictionio-parent"
-version in ThisBuild := "0.11.0-SNAPSHOT"
+version in ThisBuild := "0.11.0-incubating"
organization in ThisBuild := "org.apache.predictionio"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d5ff55fc/docs/manual/source/install/install-sourcecode.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/install-sourcecode.html.md.erb b/docs/manual/source/install/install-sourcecode.html.md.erb
index 7d5614f..adf352c 100644
--- a/docs/manual/source/install/install-sourcecode.html.md.erb
+++ b/docs/manual/source/install/install-sourcecode.html.md.erb
@@ -25,9 +25,8 @@ replace `/home/abc` with your own home directory wherever you see it.
## Downloading Source Code
Download Apache PredictionIO (incubating) <%= data.versions.pio %> from an
-Apache
-[mirror](https://www.apache.org/dyn/closer.cgi/incubator/predictionio/0.10.0-incubating/apache-predictionio-<%=
-data.versions.pio %>.tar.gz).
+Apache [mirror](https://www.apache.org/dyn/closer.cgi/incubator/predictionio/<%=
+data.versions.pio %>/apache-predictionio-<%= data.versions.pio %>.tar.gz).
## Building
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d5ff55fc/tests/pio_tests/engines/recommendation-engine/build.sbt
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/build.sbt b/tests/pio_tests/engines/recommendation-engine/build.sbt
index 13fe354..5785f39 100644
--- a/tests/pio_tests/engines/recommendation-engine/build.sbt
+++ b/tests/pio_tests/engines/recommendation-engine/build.sbt
@@ -26,6 +26,6 @@ name := "template-scala-parallel-recommendation"
organization := "org.apache.predictionio"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-SNAPSHOT" % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.11.0-incubating" % "provided",
"org.apache.spark" %% "spark-core" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided",
"org.apache.spark" %% "spark-mllib" % sys.env.getOrElse("PIO_SPARK_VERSION", "1.6.3") % "provided")
[25/50] [abbrv] incubator-predictionio git commit: Activate ES1 tests
Posted by do...@apache.org.
Activate ES1 tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/46393934
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/46393934
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/46393934
Branch: refs/heads/livedoc
Commit: 46393934815e68201c1f2732e332818eff8636de
Parents: 3da809c
Author: Donald Szeto <do...@apache.org>
Authored: Mon Mar 13 17:06:53 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 13 17:06:53 2017 -0700
----------------------------------------------------------------------
.travis.yml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/46393934/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 9549ac9..634d286 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -48,6 +48,8 @@ env:
- BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=5
- BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=5
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=1
+ - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=1
before_install:
- unset SBT_OPTS JVM_OPTS
[42/50] [abbrv] incubator-predictionio git commit: Fix a display bug
of Spark installation location
Posted by do...@apache.org.
Fix a display bug of Spark installation location
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d64b7211
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d64b7211
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d64b7211
Branch: refs/heads/livedoc
Commit: d64b7211961b8d70bedc9b239ff86dffe32707be
Parents: f8795ca
Author: Donald Szeto <do...@apache.org>
Authored: Mon Apr 3 10:49:38 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Apr 3 10:49:38 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/tools/commands/Management.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d64b7211/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index 84242ea..627b7e7 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -108,7 +108,7 @@ object Management extends EitherLogging {
info("Inspecting Apache Spark...")
val sparkHomePath = Common.getSparkHome(sparkHome)
if (new File(s"$sparkHomePath/bin/spark-submit").exists) {
- info(s"Apache Spark is installed at $sparkHome")
+ info(s"Apache Spark is installed at $sparkHomePath")
val sparkMinVersion = "1.3.0"
pioStatus = pioStatus.copy(
sparkHome = sparkHomePath,
[31/50] [abbrv] incubator-predictionio git commit: [PIO-30] Set up a
cross build for Spark 2.0 and Scala 2.11
Posted by do...@apache.org.
[PIO-30] Set up a cross build for Spark 2.0 and Scala 2.11
Closes #345
Closes #295
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/00779c3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/00779c3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/00779c3d
Branch: refs/heads/livedoc
Commit: 00779c3d8e6a7b55306ddc29d4779875ffa3ccba
Parents: bb00324
Author: Chan Lee <ch...@gmail.com>
Authored: Mon Mar 20 10:49:24 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 20 10:49:24 2017 -0700
----------------------------------------------------------------------
.travis.yml | 66 +++++-
bin/install.sh | 2 +-
build.sbt | 153 ++++++++++---
common/build.sbt | 9 +-
conf/pio-env.sh.template | 1 +
conf/pio-env.sh.travis | 5 +-
conf/pio-vendors.sh | 57 +++++
core/build.sbt | 17 +-
.../predictionio/workflow/CreateServer.scala | 3 +-
data/build.sbt | 17 +-
.../predictionio/data/api/EventServer.scala | 5 +-
.../predictionio/data/view/DataView.scala | 13 +-
.../predictionio/data/view/PBatchView.scala | 204 ++++++++++++++++++
.../data/SparkVersionDependent.scala | 30 +++
.../data/SparkVersionDependent.scala | 30 +++
e2/build.sbt | 4 +-
make-distribution.sh | 36 +---
project/Build.scala | 34 ---
project/PIOBuild.scala | 36 ++++
project/assembly.sbt | 2 +-
project/plugins.sbt | 4 +-
storage/elasticsearch/build.sbt | 30 +--
storage/elasticsearch1/build.sbt | 18 +-
storage/hbase/build.sbt | 14 +-
.../predictionio/data/view/PBatchView.scala | 212 -------------------
storage/hdfs/build.sbt | 20 +-
storage/jdbc/build.sbt | 16 +-
.../data/storage/jdbc/JDBCPEvents.scala | 26 ++-
storage/localfs/build.sbt | 16 +-
tests/Dockerfile | 23 +-
tests/Dockerfile-es1 | 57 -----
tests/README.md | 6 +
tests/after_script.travis.sh | 8 +-
tests/before_script.travis.sh | 2 +-
tests/build-docker.sh | 50 -----
tests/build_docker.sh | 57 +++++
tests/docker-compose-es1.yml | 38 ----
tests/docker-compose.yml | 2 +-
tests/docker-files/env-conf/pio-env-es1.sh | 103 ---------
tests/docker-files/env-conf/pio-env.sh | 15 +-
tests/docker-files/init.sh | 2 +-
tests/docker-files/set_build_profile.sh | 31 +++
.../engines/recommendation-engine/build.sbt | 8 +-
.../engines/recommendation-engine/manifest.json | 1 -
.../recommendation-engine/project/pio-build.sbt | 1 -
tests/pio_tests/scenarios/quickstart_test.py | 8 +-
tests/pio_tests/utils.py | 5 +-
tests/run_docker.sh | 19 +-
tests/script.travis.sh | 8 +-
tests/unit.sh | 19 +-
tools/build.sbt | 20 +-
.../predictionio/tools/admin/AdminAPI.scala | 3 +-
.../tools/export/EventsToFile.scala | 6 +-
53 files changed, 801 insertions(+), 771 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 634d286..8dcc2fa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,9 +29,6 @@ branches:
language: scala
-scala:
- - 2.10.5
-
jdk:
- oraclejdk8
@@ -44,12 +41,63 @@ cache: false
env:
matrix:
- - BUILD_TYPE=Unit METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- - BUILD_TYPE=Integration METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=5
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=5
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS ES_VERSION=1
- - BUILD_TYPE=Integration METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS ES_VERSION=1
+ - BUILD_TYPE=Unit
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ PIO_ELASTICSEARCH_VERSION=5.2.2
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ PIO_ELASTICSEARCH_VERSION=5.2.2
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.10.6
+ PIO_SPARK_VERSION=1.6.3
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+
+ - BUILD_TYPE=Unit
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ - BUILD_TYPE=Integration
+ METADATA_REP=PGSQL EVENTDATA_REP=PGSQL MODELDATA_REP=PGSQL
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ PIO_ELASTICSEARCH_VERSION=5.2.2
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ PIO_ELASTICSEARCH_VERSION=5.2.2
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=HBASE MODELDATA_REP=LOCALFS
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ PIO_ELASTICSEARCH_VERSION=1.7.3
+ - BUILD_TYPE=Integration
+ METADATA_REP=ELASTICSEARCH EVENTDATA_REP=PGSQL MODELDATA_REP=HDFS
+ PIO_SCALA_VERSION=2.11.8
+ PIO_SPARK_VERSION=2.1.0
+ PIO_ELASTICSEARCH_VERSION=1.7.3
before_install:
- unset SBT_OPTS JVM_OPTS
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/bin/install.sh
----------------------------------------------------------------------
diff --git a/bin/install.sh b/bin/install.sh
index e485df9..7431b09 100755
--- a/bin/install.sh
+++ b/bin/install.sh
@@ -18,7 +18,7 @@
#
OS=`uname`
-SPARK_VERSION=1.6.2
+SPARK_VERSION=1.6.3
# Looks like support for Elasticsearch 2.0 will require 2.0 so deferring
ELASTICSEARCH_VERSION=1.7.5
HBASE_VERSION=1.2.2
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index fa5ba29..fc47f6a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -15,28 +15,73 @@
* limitations under the License.
*/
+import PIOBuild._
import UnidocKeys._
+lazy val scalaSparkDepsVersion = Map(
+ "2.10" -> Map(
+ "1.6" -> Map(
+ "akka" -> "2.3.15",
+ "hadoop" -> "2.6.5",
+ "json4s" -> "3.2.10"),
+ "2.0" -> Map(
+ "akka" -> "2.3.16",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.1" -> Map(
+ "akka" -> "2.3.16",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11")),
+ "2.11" -> Map(
+ "1.6" -> Map(
+ "akka" -> "2.3.15",
+ "hadoop" -> "2.6.5",
+ "json4s" -> "3.2.10"),
+ "2.0" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11"),
+ "2.1" -> Map(
+ "akka" -> "2.4.17",
+ "hadoop" -> "2.7.3",
+ "json4s" -> "3.2.11")))
+
name := "apache-predictionio-parent"
version in ThisBuild := "0.11.0-SNAPSHOT"
organization in ThisBuild := "org.apache.predictionio"
-scalaVersion in ThisBuild := "2.10.5"
+scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.10.6")
+
+crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
scalacOptions in (ThisBuild, Test) ++= Seq("-Yrangepos")
-
fork in (ThisBuild, run) := true
javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7",
"-Xlint:deprecation", "-Xlint:unchecked")
-json4sVersion in ThisBuild := "3.2.10"
+// Ignore differentiation of Spark patch levels
+sparkVersion in ThisBuild := sys.props.getOrElse("spark.version", "1.6.3")
+
+sparkBinaryVersion in ThisBuild := binaryVersion(sparkVersion.value)
+
+akkaVersion in ThisBuild := sys.props.getOrElse(
+ "akka.version",
+ scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("akka"))
+
+lazy val es = sys.props.getOrElse("elasticsearch.version", "1.7.6")
-sparkVersion in ThisBuild := "1.6.3"
+elasticsearchVersion in ThisBuild := es
+
+json4sVersion in ThisBuild := scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("json4s")
+
+hadoopVersion in ThisBuild := sys.props.getOrElse(
+ "hadoop.version",
+ scalaSparkDepsVersion(scalaBinaryVersion.value)(sparkBinaryVersion.value)("hadoop"))
val pioBuildInfoSettings = buildInfoSettings ++ Seq(
sourceGenerators in Compile <+= buildInfo,
@@ -45,65 +90,106 @@ val pioBuildInfoSettings = buildInfoSettings ++ Seq(
version,
scalaVersion,
sbtVersion,
- sparkVersion),
+ sparkVersion,
+ hadoopVersion),
buildInfoPackage := "org.apache.predictionio.core")
+// Used temporarily to modify genjavadoc version to "0.10" until unidoc updates it
+val genjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
+ libraryDependencies += compilerPlugin("com.typesafe.genjavadoc" %% "genjavadoc-plugin" % "0.10" cross CrossVersion.full),
+ scalacOptions <+= target map (t => "-P:genjavadoc:out=" + (t / "java")))
+
val conf = file("conf")
val commonSettings = Seq(
autoAPIMappings := true,
- unmanagedClasspath in Test += conf)
+ unmanagedClasspath in Test += conf,
+ unmanagedClasspath in Test += baseDirectory.value.getParentFile / s"storage/jdbc/target/scala-${scalaBinaryVersion.value}/classes")
-val common = (project in file("common")).
- settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
-
-val data = (project in file("data")).
- dependsOn(common).
- settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+val commonTestSettings = Seq(
+ libraryDependencies ++= Seq(
+ "org.postgresql" % "postgresql" % "9.4-1204-jdbc41" % "test",
+ "org.scalikejdbc" %% "scalikejdbc" % "2.3.5" % "test"))
val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataElasticsearch = (project in file("storage/elasticsearch")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataHbase = (project in file("storage/hbase")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataHdfs = (project in file("storage/hdfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataJdbc = (project in file("storage/jdbc")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
val dataLocalfs = (project in file("storage/localfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ settings(publishArtifact := false)
+
+val common = (project in file("common")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*).
+ disablePlugins(sbtassembly.AssemblyPlugin)
+
+val data = (project in file("data")).
+ dependsOn(common).
+ settings(commonSettings: _*).
+ settings(commonTestSettings: _*).
+ settings(genjavadocSettings: _*).
+ settings(unmanagedSourceDirectories in Compile +=
+ sourceDirectory.value / s"main/spark-${majorVersion(sparkVersion.value)}").
+ disablePlugins(sbtassembly.AssemblyPlugin)
val core = (project in file("core")).
dependsOn(data).
settings(commonSettings: _*).
+ settings(commonTestSettings: _*).
settings(genjavadocSettings: _*).
settings(pioBuildInfoSettings: _*).
- enablePlugins(SbtTwirl)
+ enablePlugins(SbtTwirl).
+ disablePlugins(sbtassembly.AssemblyPlugin)
val tools = (project in file("tools")).
dependsOn(core).
dependsOn(data).
settings(commonSettings: _*).
+ settings(commonTestSettings: _*).
settings(genjavadocSettings: _*).
- enablePlugins(SbtTwirl)
+ enablePlugins(SbtTwirl).
+ settings(publishArtifact := false)
val e2 = (project in file("e2")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*)
+ settings(genjavadocSettings: _*).
+ disablePlugins(sbtassembly.AssemblyPlugin)
+
+val dataEs = if (majorVersion(es) == 1) dataElasticsearch1 else dataElasticsearch
+
+val storageSubprojects = Seq(
+ dataEs,
+ dataHbase,
+ dataHdfs,
+ dataJdbc,
+ dataLocalfs)
+
+val storage = (project in file("storage"))
+ .aggregate(storageSubprojects map Project.projectToRef: _*)
+ .disablePlugins(sbtassembly.AssemblyPlugin)
val root = (project in file(".")).
settings(commonSettings: _*).
@@ -162,18 +248,8 @@ val root = (project in file(".")).
"docs/javadoc/javadoc-overview.html",
"-noqualifier",
"java.lang")).
- aggregate(
- common,
- core,
- data,
- dataElasticsearch1,
- dataElasticsearch,
- dataHbase,
- dataHdfs,
- dataJdbc,
- dataLocalfs,
- tools,
- e2)
+ aggregate(common, core, data, tools, e2).
+ disablePlugins(sbtassembly.AssemblyPlugin)
val pioUnidoc = taskKey[Unit]("Builds PredictionIO ScalaDoc")
@@ -232,3 +308,10 @@ parallelExecution := false
parallelExecution in Global := false
testOptions in Test += Tests.Argument("-oDF")
+
+printBuildInfo := {
+ println(s"PIO_SCALA_VERSION=${scalaVersion.value}")
+ println(s"PIO_SPARK_VERSION=${sparkVersion.value}")
+ println(s"PIO_ELASTICSEARCH_VERSION=${elasticsearchVersion.value}")
+ println(s"PIO_HADOOP_VERSION=${hadoopVersion.value}")
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/common/build.sbt
----------------------------------------------------------------------
diff --git a/common/build.sbt b/common/build.sbt
index e7050d4..5a0fba1 100644
--- a/common/build.sbt
+++ b/common/build.sbt
@@ -15,12 +15,15 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-common"
libraryDependencies ++= Seq(
- "io.spray" %% "spray-can" % "1.3.2",
- "io.spray" %% "spray-routing" % "1.3.2",
- "org.spark-project.akka" %% "akka-actor" % "2.3.4-spark"
+ "io.spray" %% "spray-can" % "1.3.3",
+ "io.spray" %% "spray-routing" % "1.3.3",
+ "com.typesafe.akka" %% "akka-actor" % akkaVersion.value,
+ "com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value
)
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/conf/pio-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.template b/conf/pio-env.sh.template
index 0d76102..57185ba 100644
--- a/conf/pio-env.sh.template
+++ b/conf/pio-env.sh.template
@@ -24,6 +24,7 @@
# you need to change these to fit your site.
# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
+# SPARK_HOME=$PIO_HOME/vendors/spark-2.0.2-bin-hadoop2.7
SPARK_HOME=$PIO_HOME/vendors/spark-1.6.3-bin-hadoop2.6
POSTGRES_JDBC_DRIVER=$PIO_HOME/lib/postgresql-9.4-1204.jdbc41.jar
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/conf/pio-env.sh.travis
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.travis b/conf/pio-env.sh.travis
index 94b30cd..80e3332 100644
--- a/conf/pio-env.sh.travis
+++ b/conf/pio-env.sh.travis
@@ -24,7 +24,8 @@
# you need to change these to fit your site.
# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$PIO_HOME/vendors/spark-1.3.0-bin-hadoop2.4
+# it is set up in script.travis.sh
+SPARK_HOME=$SPARK_HOME
# Filesystem paths where PredictionIO uses as block storage.
PIO_FS_BASEDIR=$HOME/.pio_store
@@ -41,7 +42,7 @@ PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/models
PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
-PIO_STORAGE_SOURCES_HBASE_HOME=$PIO_HOME/vendors/hbase-1.0.0
+PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME
# Storage Data Sources (pgsql)
PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/conf/pio-vendors.sh
----------------------------------------------------------------------
diff --git a/conf/pio-vendors.sh b/conf/pio-vendors.sh
new file mode 100644
index 0000000..830b576
--- /dev/null
+++ b/conf/pio-vendors.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# IMPORTANT: PIO_*_VERSION for dependencies must be set before envoking this script.
+# `source conf/set_build_profile.sh $BUILD_PROFILE` to get the proper versions
+
+if [ -z "$PIO_SCALA_VERSION" ]; then
+ PIO_SCALA_VERSION="2.10.6"
+fi
+
+if [ -z "$PIO_SPARK_VERSION" ]; then
+ PIO_SPARK_VERSION="1.6.3"
+fi
+
+if [ -z "$PIO_HADOOP_VERSION" ]; then
+ PIO_HADOOP_VERSION="2.6.5"
+fi
+
+if [ -z "$PIO_ELASTICSEARCH_VERSION" ]; then
+ PIO_ELASTICSEARCH_VERSION="1.7.3"
+fi
+
+ES_MAJOR=`echo $PIO_ELASTICSEARCH_VERSION | awk -F. '{print $1}'`
+
+if [ "$ES_MAJOR" = "1" ]; then
+ export ES_IMAGE="elasticsearch"
+ export ES_TAG="1"
+else
+ export ES_IMAGE="docker.elastic.co/elasticsearch/elasticsearch"
+ export ES_TAG="5.2.2"
+fi
+
+PGSQL_JAR=postgresql-9.4-1204.jdbc41.jar
+PGSQL_DOWNLOAD=https://jdbc.postgresql.org/download/${PGSQL_JAR}
+
+HADOOP_MAJOR=`echo $PIO_HADOOP_VERSION | awk -F. '{print $1 "." $2}'`
+SPARK_DIR=spark-${PIO_SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR}
+SPARK_ARCHIVE=${SPARK_DIR}.tgz
+SPARK_DOWNLOAD=http://d3kbcqa49mib13.cloudfront.net/${SPARK_ARCHIVE}
+# ELASTICSEARCH_DOWNLOAD
+# 5.x https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
+# 1.x https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${PIO_ELASTICSEARCH_VERSION}.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
index bfb8bf3..b95a957 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -15,32 +15,23 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-core"
libraryDependencies ++= Seq(
- "com.github.scopt" %% "scopt" % "3.3.0",
+ "com.github.scopt" %% "scopt" % "3.5.0",
"com.google.code.gson" % "gson" % "2.5",
- "com.google.guava" % "guava" % "18.0",
- "com.twitter" %% "chill" % "0.7.2"
- exclude("com.esotericsoftware.minlog", "minlog"),
"com.twitter" %% "chill-bijection" % "0.7.2",
"de.javakaffee" % "kryo-serializers" % "0.37",
- "commons-io" % "commons-io" % "2.4",
- "io.spray" %% "spray-can" % "1.3.3",
- "io.spray" %% "spray-routing" % "1.3.3",
"net.jodah" % "typetools" % "0.3.1",
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
- "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
- "org.clapper" %% "grizzled-slf4j" % "1.0.2",
- "org.json4s" %% "json4s-native" % json4sVersion.value,
"org.json4s" %% "json4s-ext" % json4sVersion.value,
"org.scalaj" %% "scalaj-http" % "1.1.6",
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
"org.slf4j" % "slf4j-log4j12" % "1.7.18",
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
-//testOptions := Seq(Tests.Filter(s => Seq("Dev").exists(s.contains(_))))
-
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
index 6f274bc..31b7831 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
@@ -51,7 +51,6 @@ import spray.routing._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scala.concurrent.duration._
-import scala.concurrent.future
import scala.language.existentials
import scala.util.{Failure, Random, Success}
import scalaj.http.HttpOptions
@@ -548,7 +547,7 @@ class ServerActor[Q, P](
"prediction" -> prediction)) ++ queryPrId
// At this point args.accessKey should be Some(String).
val accessKey = args.accessKey.getOrElse("")
- val f: Future[Int] = future {
+ val f: Future[Int] = Future {
scalaj.http.Http(
s"http://${args.eventServerIp}:${args.eventServerPort}/" +
s"events.json?accessKey=$accessKey").postData(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index f5e95b5..ddd085d 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -15,28 +15,21 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data"
libraryDependencies ++= Seq(
"com.github.nscala-time" %% "nscala-time" % "2.6.0",
- "commons-codec" % "commons-codec" % "1.9",
+ "com.google.guava" % "guava" % "19.0",
"io.spray" %% "spray-can" % "1.3.3",
"io.spray" %% "spray-routing" % "1.3.3",
- "io.spray" %% "spray-testkit" % "1.3.3" % "test",
- "mysql" % "mysql-connector-java" % "5.1.37" % "optional",
- "org.apache.hadoop" % "hadoop-common" % "2.6.2"
- exclude("javax.servlet", "servlet-api"),
- "org.apache.zookeeper" % "zookeeper" % "3.4.7"
- exclude("org.slf4j", "slf4j-api")
- exclude("org.slf4j", "slf4j-log4j12"),
- "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
"org.json4s" %% "json4s-native" % json4sVersion.value,
- "org.json4s" %% "json4s-ext" % json4sVersion.value,
+ "org.scalikejdbc" %% "scalikejdbc" % "2.3.2",
+ "io.spray" %% "spray-testkit" % "1.3.3" % "test",
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.slf4j" % "slf4j-log4j12" % "1.7.18",
- "org.spark-project.akka" %% "akka-actor" % "2.3.4-spark",
"org.specs2" %% "specs2" % "2.3.13" % "test")
parallelExecution in Test := false
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
index 648316e..b4392ff 100644
--- a/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/api/EventServer.scala
@@ -48,8 +48,7 @@ import spray.httpx.Json4sSupport
import spray.routing._
import spray.routing.authentication.Authentication
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
+import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Try, Success, Failure}
class EventServiceActor(
@@ -635,7 +634,7 @@ object EventServer {
}
object Run {
- def main(args: Array[String]) {
+ def main(args: Array[String]): Unit = {
EventServer.createEventServer(EventServerConfig(
ip = "0.0.0.0",
port = 7070))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala b/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
index 4866b5d..1c47e10 100644
--- a/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/view/DataView.scala
@@ -20,6 +20,7 @@ package org.apache.predictionio.data.view
import org.apache.predictionio.annotation.Experimental
import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.SparkVersionDependent
import grizzled.slf4j.Logger
import org.apache.predictionio.data.store.PEventStore
@@ -27,7 +28,7 @@ import org.apache.predictionio.data.store.PEventStore
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.SaveMode
-import org.apache.spark.sql.SQLContext
+import org.apache.spark.SparkContext
import org.joda.time.DateTime
import scala.reflect.ClassTag
@@ -64,11 +65,11 @@ object DataView {
untilTime: Option[DateTime] = None,
conversionFunction: Event => Option[E],
name: String = "",
- version: String = "")(sqlContext: SQLContext): DataFrame = {
+ version: String = "")(sc: SparkContext): DataFrame = {
@transient lazy val logger = Logger[this.type]
- val sc = sqlContext.sparkContext
+ val sqlSession = SparkVersionDependent.sqlSession(sc)
val beginTime = startTime match {
case Some(t) => t
@@ -85,7 +86,7 @@ object DataView {
val baseDir = s"${sys.env("PIO_FS_BASEDIR")}/view"
val fileName = s"$baseDir/$name-$appName-$hash.parquet"
try {
- sqlContext.read.parquet(fileName)
+ sqlSession.read.parquet(fileName)
} catch {
case e: java.io.FileNotFoundException =>
logger.info("Cached copy not found, reading from DB.")
@@ -96,11 +97,11 @@ object DataView {
startTime = startTime,
untilTime = Some(endTime))(sc)
.flatMap((e) => conversionFunction(e))
- import sqlContext.implicits._ // needed for RDD.toDF()
+ import sqlSession.implicits._ // needed for RDD.toDF()
val resultDF = result.toDF()
resultDF.write.mode(SaveMode.ErrorIfExists).parquet(fileName)
- sqlContext.read.parquet(fileName)
+ sqlSession.read.parquet(fileName)
case e: java.lang.RuntimeException =>
if (e.toString.contains("is not a Parquet file")) {
logger.error(s"$fileName does not contain a valid Parquet file. " +
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala b/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
new file mode 100644
index 0000000..cbdebce
--- /dev/null
+++ b/data/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.view
+
+import org.apache.predictionio.data.storage.{DataMap, Event, EventValidation, Storage}
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.joda.time.DateTime
+import org.json4s.JValue
+
+
+// each JValue data associated with the time it is set
+private[predictionio] case class PropTime(val d: JValue, val t: Long) extends Serializable
+
+private[predictionio] case class SetProp (
+ val fields: Map[String, PropTime],
+ // last set time. Note: fields could be empty with valid set time
+ val t: Long) extends Serializable {
+
+ def ++ (that: SetProp): SetProp = {
+ val commonKeys = fields.keySet.intersect(that.fields.keySet)
+
+ val common: Map[String, PropTime] = commonKeys.map { k =>
+ val thisData = this.fields(k)
+ val thatData = that.fields(k)
+ // only keep the value with latest time
+ val v = if (thisData.t > thatData.t) thisData else thatData
+ (k, v)
+ }.toMap
+
+ val combinedFields = common ++
+ (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
+
+ // keep the latest set time
+ val combinedT = if (this.t > that.t) this.t else that.t
+
+ SetProp(
+ fields = combinedFields,
+ t = combinedT
+ )
+ }
+}
+
+private[predictionio] case class UnsetProp (fields: Map[String, Long]) extends Serializable {
+ def ++ (that: UnsetProp): UnsetProp = {
+ val commonKeys = fields.keySet.intersect(that.fields.keySet)
+
+ val common: Map[String, Long] = commonKeys.map { k =>
+ val thisData = this.fields(k)
+ val thatData = that.fields(k)
+ // only keep the value with latest time
+ val v = if (thisData > thatData) thisData else thatData
+ (k, v)
+ }.toMap
+
+ val combinedFields = common ++
+ (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
+
+ UnsetProp(
+ fields = combinedFields
+ )
+ }
+}
+
+private[predictionio] case class DeleteEntity (t: Long) extends Serializable {
+ def ++ (that: DeleteEntity): DeleteEntity = {
+ if (this.t > that.t) this else that
+ }
+}
+
+private[predictionio] case class EventOp (
+ val setProp: Option[SetProp] = None,
+ val unsetProp: Option[UnsetProp] = None,
+ val deleteEntity: Option[DeleteEntity] = None
+) extends Serializable {
+
+ def ++ (that: EventOp): EventOp = {
+ EventOp(
+ setProp = (setProp ++ that.setProp).reduceOption(_ ++ _),
+ unsetProp = (unsetProp ++ that.unsetProp).reduceOption(_ ++ _),
+ deleteEntity = (deleteEntity ++ that.deleteEntity).reduceOption(_ ++ _)
+ )
+ }
+
+ def toDataMap(): Option[DataMap] = {
+ setProp.flatMap { set =>
+
+ val unsetKeys: Set[String] = unsetProp.map( unset =>
+ unset.fields.filter{ case (k, v) => (v >= set.fields(k).t) }.keySet
+ ).getOrElse(Set())
+
+ val combinedFields = deleteEntity.map { delete =>
+ if (delete.t >= set.t) {
+ None
+ } else {
+ val deleteKeys: Set[String] = set.fields
+ .filter { case (k, PropTime(kv, t)) =>
+ (delete.t >= t)
+ }.keySet
+ Some(set.fields -- unsetKeys -- deleteKeys)
+ }
+ }.getOrElse{
+ Some(set.fields -- unsetKeys)
+ }
+
+ // Note: mapValues() doesn't return concrete Map and causes
+ // NotSerializableException issue. Use map(identity) to work around this.
+ // see https://issues.scala-lang.org/browse/SI-7005
+ combinedFields.map(f => DataMap(f.mapValues(_.d).map(identity)))
+ }
+ }
+
+}
+
+private[predictionio] object EventOp {
+ def apply(e: Event): EventOp = {
+ val t = e.eventTime.getMillis
+ e.event match {
+ case "$set" => {
+ val fields = e.properties.fields.mapValues(jv =>
+ PropTime(jv, t)
+ ).map(identity)
+
+ EventOp(
+ setProp = Some(SetProp(fields = fields, t = t))
+ )
+ }
+ case "$unset" => {
+ val fields = e.properties.fields.mapValues(jv => t).map(identity)
+ EventOp(
+ unsetProp = Some(UnsetProp(fields = fields))
+ )
+ }
+ case "$delete" => {
+ EventOp(
+ deleteEntity = Some(DeleteEntity(t))
+ )
+ }
+ case _ => {
+ EventOp()
+ }
+ }
+ }
+}
+
+@deprecated("Use PEvents or PEventStore instead.", "0.9.2")
+class PBatchView(
+ val appId: Int,
+ val startTime: Option[DateTime],
+ val untilTime: Option[DateTime],
+ val sc: SparkContext) {
+
+ // NOTE: parallel Events DB interface
+ @transient lazy val eventsDb = Storage.getPEvents()
+
+ @transient lazy val _events: RDD[Event] =
+ eventsDb.getByAppIdAndTimeAndEntity(
+ appId = appId,
+ startTime = startTime,
+ untilTime = untilTime,
+ entityType = None,
+ entityId = None)(sc)
+
+ // TODO: change to use EventSeq?
+ @transient lazy val events: RDD[Event] = _events
+
+ def aggregateProperties(
+ entityType: String,
+ startTimeOpt: Option[DateTime] = None,
+ untilTimeOpt: Option[DateTime] = None
+ ): RDD[(String, DataMap)] = {
+
+ _events
+ .filter( e => ((e.entityType == entityType) &&
+ (EventValidation.isSpecialEvents(e.event))) )
+ .map( e => (e.entityId, EventOp(e) ))
+ .aggregateByKey[EventOp](EventOp())(
+ // within same partition
+ seqOp = { case (u, v) => u ++ v },
+ // across partition
+ combOp = { case (accu, u) => accu ++ u }
+ )
+ .mapValues(_.toDataMap)
+ .filter{ case (k, v) => v.isDefined }
+ .map{ case (k, v) => (k, v.get) }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
----------------------------------------------------------------------
diff --git a/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala b/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
new file mode 100644
index 0000000..0652e0b
--- /dev/null
+++ b/data/src/main/spark-1/org/apache/predictionio/data/SparkVersionDependent.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data
+
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.SQLContext
+
+object SparkVersionDependent {
+
+ def sqlSession(sc: SparkContext): SQLContext = {
+ return new SQLContext(sc)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/data/src/main/spark-2/org/apache/predictionio/data/SparkVersionDependent.scala
----------------------------------------------------------------------
diff --git a/data/src/main/spark-2/org/apache/predictionio/data/SparkVersionDependent.scala b/data/src/main/spark-2/org/apache/predictionio/data/SparkVersionDependent.scala
new file mode 100644
index 0000000..3d07bdf
--- /dev/null
+++ b/data/src/main/spark-2/org/apache/predictionio/data/SparkVersionDependent.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data
+
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.SparkSession
+
+object SparkVersionDependent {
+
+ def sqlSession(sc: SparkContext): SparkSession = {
+ SparkSession.builder().getOrCreate()
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/e2/build.sbt
----------------------------------------------------------------------
diff --git a/e2/build.sbt b/e2/build.sbt
index 50de5e5..0774a55 100644
--- a/e2/build.sbt
+++ b/e2/build.sbt
@@ -15,14 +15,14 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-e2"
parallelExecution in Test := false
libraryDependencies ++= Seq(
- "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.spark" %% "spark-mllib" % sparkVersion.value % "provided",
- "org.clapper" %% "grizzled-slf4j" % "1.0.2",
"org.scalatest" %% "scalatest" % "2.2.5" % "test")
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index c360c0e..e92178f 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -21,15 +21,12 @@ set -e
usage ()
{
- echo "Usage: $0 [-h|--help] [--with-es=x]"
+ echo "Usage: $0 [-h|--help]"
echo ""
echo " -h|--help Show usage"
- echo ""
- echo " --with-es=1 Build distribution with Elasticsearch 1 support as default"
- echo " --with-es=5 Build distribution with Elasticsearch 5 support as default"
}
-ES_VERSION=1
+JAVA_PROPS=()
for i in "$@"
do
@@ -39,8 +36,8 @@ case $i in
shift
exit
;;
- --with-es=*)
- ES_VERSION="${i#*=}"
+ -D*)
+ JAVA_PROPS+=("$i")
shift
;;
*)
@@ -50,23 +47,19 @@ case $i in
esac
done
-if [ "$ES_VERSION" = "1" ] || [ "$ES_VERSION" = "5" ]
-then
- echo -e "\033[0;32mBuilding with Elasticsearch $ES_VERSION support as the default choice\033[0m"
-else
- usage
- exit 1
-fi
-
FWDIR="$(cd `dirname $0`; pwd)"
DISTDIR="${FWDIR}/dist"
-VERSION=$(grep version ${FWDIR}/build.sbt | grep ThisBuild | grep -o '".*"' | sed 's/"//g')
+VERSION=$(grep ^version ${FWDIR}/build.sbt | grep ThisBuild | grep -o '".*"' | sed 's/"//g')
echo "Building binary distribution for PredictionIO $VERSION..."
cd ${FWDIR}
-sbt/sbt common/publishLocal data/publishLocal core/publishLocal e2/publishLocal dataElasticsearch1/assembly dataElasticsearch/assembly dataHbase/assembly dataHdfs/assembly dataJdbc/assembly dataLocalfs/assembly tools/assembly
+set -x
+sbt/sbt "${JAVA_PROPS[@]}" clean
+sbt/sbt "${JAVA_PROPS[@]}" printBuildInfo
+sbt/sbt "${JAVA_PROPS[@]}" publishLocal assembly storage/assembly
+set +x
cd ${FWDIR}
rm -rf ${DISTDIR}
@@ -74,8 +67,8 @@ mkdir -p ${DISTDIR}/bin
mkdir -p ${DISTDIR}/conf
mkdir -p ${DISTDIR}/lib
mkdir -p ${DISTDIR}/lib/spark
-mkdir -p ${DISTDIR}/lib/extra
mkdir -p ${DISTDIR}/project
+
mkdir -p ${DISTDIR}/sbt
cp ${FWDIR}/bin/* ${DISTDIR}/bin || :
@@ -85,13 +78,6 @@ cp ${FWDIR}/sbt/sbt ${DISTDIR}/sbt
cp ${FWDIR}/assembly/*assembly*jar ${DISTDIR}/lib
cp ${FWDIR}/assembly/spark/*jar ${DISTDIR}/lib/spark
-if [ "$ES_VERSION" = "5" ]
-then
- mv ${DISTDIR}/lib/spark/pio-data-elasticsearch1-assembly-*.jar ${DISTDIR}/lib/extra
-else
- mv ${DISTDIR}/lib/spark/pio-data-elasticsearch-assembly-*.jar ${DISTDIR}/lib/extra
-fi
-
rm -f ${DISTDIR}/lib/*javadoc.jar
rm -f ${DISTDIR}/lib/*sources.jar
rm -f ${DISTDIR}/conf/pio-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
deleted file mode 100644
index 885073a..0000000
--- a/project/Build.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt._
-import Keys._
-
-object PIOBuild extends Build {
- val elasticsearchVersion = SettingKey[String](
- "elasticsearch-version",
- "The version of Elasticsearch used for building.")
- val json4sVersion = SettingKey[String](
- "json4s-version",
- "The version of JSON4S used for building.")
- val sparkVersion = SettingKey[String](
- "spark-version",
- "The version of Apache Spark used for building.")
- val childrenPomExtra = SettingKey[scala.xml.NodeSeq](
- "children-pom-extra",
- "Extra POM data for children projects.")
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/project/PIOBuild.scala
----------------------------------------------------------------------
diff --git a/project/PIOBuild.scala b/project/PIOBuild.scala
new file mode 100644
index 0000000..30fca65
--- /dev/null
+++ b/project/PIOBuild.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt._
+
+object PIOBuild {
+ val elasticsearchVersion = settingKey[String]("The version of Elasticsearch used for building")
+ val json4sVersion = settingKey[String]("The version of JSON4S used for building")
+ val sparkVersion = settingKey[String]("The version of Apache Spark used for building")
+ val sparkBinaryVersion = settingKey[String]("The binary version of Apache Spark used for building")
+ val hadoopVersion = settingKey[String]("The version of Apache Hadoop used for building")
+ val akkaVersion = settingKey[String]("The version of Akka used for building")
+
+ val childrenPomExtra = settingKey[scala.xml.NodeSeq]("Extra POM data for children projects")
+ val elasticsearchSparkArtifact = settingKey[String]("Name of Elasticsearch-Spark artifact used for building")
+
+ def binaryVersion(versionString: String): String = versionString.split('.').take(2).mkString(".")
+ def majorVersion(versionString: String): Int = versionString.split('.')(0).toInt
+ def minorVersion(versionString: String): Int = versionString.split('.')(1).toInt
+
+ lazy val printBuildInfo = taskKey[Unit]("Print build information")
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/project/assembly.sbt
----------------------------------------------------------------------
diff --git a/project/assembly.sbt b/project/assembly.sbt
index 49085ee..39c1bb8 100644
--- a/project/assembly.sbt
+++ b/project/assembly.sbt
@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.1")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 3edaf67..2f21e00 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -2,7 +2,7 @@ addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.3.2")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
-addSbtPlugin("com.typesafe.sbt" % "sbt-twirl" % "1.0.3")
+addSbtPlugin("com.typesafe.sbt" % "sbt-twirl" % "1.1.1")
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
@@ -10,4 +10,4 @@ addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositories/releases/"
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 7d9b2ad..52dc965 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -15,36 +15,26 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-elasticsearch"
-elasticsearchVersion := "5.2.1"
+elasticsearchSparkArtifact := (if (majorVersion(sparkVersion.value) == 2) "elasticsearch-spark-20" else "elasticsearch-spark-13")
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
- "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
- "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
- "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
- "org.elasticsearch.client" % "rest" % elasticsearchVersion.value,
- "org.elasticsearch" %% "elasticsearch-spark-13" % elasticsearchVersion.value
- exclude("org.apache.spark", "spark-sql_2.10")
- exclude("org.apache.spark", "spark-streaming_2.10"),
- "org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value,
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
+ "org.elasticsearch.client" % "rest" % elasticsearchVersion.value,
+ "org.elasticsearch" %% elasticsearchSparkArtifact.value % elasticsearchVersion.value
+ exclude("org.apache.spark", "*"),
+ "org.elasticsearch" % "elasticsearch-hadoop-mr" % elasticsearchVersion.value,
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
-
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
assemblyShadeRules in assembly := Seq(
ShadeRule.rename("org.apache.http.**" -> "shadeio.data.http.@1").inAll
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index dde7285..bb68b11 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -15,30 +15,20 @@
* limitations under the License.
*/
-name := "apache-predictionio-data-elasticsearch1"
+import PIOBuild._
-elasticsearchVersion := "1.7.3"
+name := "apache-predictionio-data-elasticsearch1"
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
- "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
"org.elasticsearch" % "elasticsearch" % elasticsearchVersion.value,
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
-
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 513e294..19aa126 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -15,11 +15,12 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-hbase"
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
- "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.hbase" % "hbase-common" % "0.98.5-hadoop2",
"org.apache.hbase" % "hbase-client" % "0.98.5-hadoop2"
@@ -32,22 +33,13 @@ libraryDependencies ++= Seq(
exclude("org.mortbay.jetty", "servlet-api-2.5")
exclude("org.mortbay.jetty", "jsp-api-2.1")
exclude("org.mortbay.jetty", "jsp-2.1"),
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
-
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
deleted file mode 100644
index b453820..0000000
--- a/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.view
-
-import org.apache.predictionio.data.storage.hbase.HBPEvents
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventValidation
-import org.apache.predictionio.data.storage.DataMap
-import org.apache.predictionio.data.storage.Storage
-
-import org.joda.time.DateTime
-
-import org.json4s.JValue
-
-import org.apache.spark.SparkContext
-import org.apache.spark.SparkContext._
-import org.apache.spark.rdd.RDD
-
-
-// each JValue data associated with the time it is set
-private[predictionio] case class PropTime(val d: JValue, val t: Long) extends Serializable
-
-private[predictionio] case class SetProp (
- val fields: Map[String, PropTime],
- // last set time. Note: fields could be empty with valid set time
- val t: Long) extends Serializable {
-
- def ++ (that: SetProp): SetProp = {
- val commonKeys = fields.keySet.intersect(that.fields.keySet)
-
- val common: Map[String, PropTime] = commonKeys.map { k =>
- val thisData = this.fields(k)
- val thatData = that.fields(k)
- // only keep the value with latest time
- val v = if (thisData.t > thatData.t) thisData else thatData
- (k, v)
- }.toMap
-
- val combinedFields = common ++
- (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
-
- // keep the latest set time
- val combinedT = if (this.t > that.t) this.t else that.t
-
- SetProp(
- fields = combinedFields,
- t = combinedT
- )
- }
-}
-
-private[predictionio] case class UnsetProp (fields: Map[String, Long]) extends Serializable {
- def ++ (that: UnsetProp): UnsetProp = {
- val commonKeys = fields.keySet.intersect(that.fields.keySet)
-
- val common: Map[String, Long] = commonKeys.map { k =>
- val thisData = this.fields(k)
- val thatData = that.fields(k)
- // only keep the value with latest time
- val v = if (thisData > thatData) thisData else thatData
- (k, v)
- }.toMap
-
- val combinedFields = common ++
- (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
-
- UnsetProp(
- fields = combinedFields
- )
- }
-}
-
-private[predictionio] case class DeleteEntity (t: Long) extends Serializable {
- def ++ (that: DeleteEntity): DeleteEntity = {
- if (this.t > that.t) this else that
- }
-}
-
-private[predictionio] case class EventOp (
- val setProp: Option[SetProp] = None,
- val unsetProp: Option[UnsetProp] = None,
- val deleteEntity: Option[DeleteEntity] = None
-) extends Serializable {
-
- def ++ (that: EventOp): EventOp = {
- EventOp(
- setProp = (setProp ++ that.setProp).reduceOption(_ ++ _),
- unsetProp = (unsetProp ++ that.unsetProp).reduceOption(_ ++ _),
- deleteEntity = (deleteEntity ++ that.deleteEntity).reduceOption(_ ++ _)
- )
- }
-
- def toDataMap(): Option[DataMap] = {
- setProp.flatMap { set =>
-
- val unsetKeys: Set[String] = unsetProp.map( unset =>
- unset.fields.filter{ case (k, v) => (v >= set.fields(k).t) }.keySet
- ).getOrElse(Set())
-
- val combinedFields = deleteEntity.map { delete =>
- if (delete.t >= set.t) {
- None
- } else {
- val deleteKeys: Set[String] = set.fields
- .filter { case (k, PropTime(kv, t)) =>
- (delete.t >= t)
- }.keySet
- Some(set.fields -- unsetKeys -- deleteKeys)
- }
- }.getOrElse{
- Some(set.fields -- unsetKeys)
- }
-
- // Note: mapValues() doesn't return concrete Map and causes
- // NotSerializableException issue. Use map(identity) to work around this.
- // see https://issues.scala-lang.org/browse/SI-7005
- combinedFields.map(f => DataMap(f.mapValues(_.d).map(identity)))
- }
- }
-
-}
-
-private[predictionio] object EventOp {
- def apply(e: Event): EventOp = {
- val t = e.eventTime.getMillis
- e.event match {
- case "$set" => {
- val fields = e.properties.fields.mapValues(jv =>
- PropTime(jv, t)
- ).map(identity)
-
- EventOp(
- setProp = Some(SetProp(fields = fields, t = t))
- )
- }
- case "$unset" => {
- val fields = e.properties.fields.mapValues(jv => t).map(identity)
- EventOp(
- unsetProp = Some(UnsetProp(fields = fields))
- )
- }
- case "$delete" => {
- EventOp(
- deleteEntity = Some(DeleteEntity(t))
- )
- }
- case _ => {
- EventOp()
- }
- }
- }
-}
-
-@deprecated("Use PEvents or PEventStore instead.", "0.9.2")
-class PBatchView(
- val appId: Int,
- val startTime: Option[DateTime],
- val untilTime: Option[DateTime],
- val sc: SparkContext) {
-
- // NOTE: parallel Events DB interface
- @transient lazy val eventsDb = Storage.getPEvents()
-
- @transient lazy val _events: RDD[Event] =
- eventsDb.getByAppIdAndTimeAndEntity(
- appId = appId,
- startTime = startTime,
- untilTime = untilTime,
- entityType = None,
- entityId = None)(sc)
-
- // TODO: change to use EventSeq?
- @transient lazy val events: RDD[Event] = _events
-
- def aggregateProperties(
- entityType: String,
- startTimeOpt: Option[DateTime] = None,
- untilTimeOpt: Option[DateTime] = None
- ): RDD[(String, DataMap)] = {
-
- _events
- .filter( e => ((e.entityType == entityType) &&
- (EventValidation.isSpecialEvents(e.event))) )
- .map( e => (e.entityId, EventOp(e) ))
- .aggregateByKey[EventOp](EventOp())(
- // within same partition
- seqOp = { case (u, v) => u ++ v },
- // across partition
- combOp = { case (accu, u) => accu ++ u }
- )
- .mapValues(_.toDataMap)
- .filter{ case (k, v) => v.isDefined }
- .map{ case (k, v) => (k, v.get) }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 7ddc86c..8bae2a8 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -15,27 +15,25 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-hdfs"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion.value
+ exclude("commons-beanutils", "*"),
"org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(
+ includeScala = false,
+ excludedJars = (fullClasspath in assembly).value.filter {_.data.getName startsWith "apache-predictionio"})
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index 2011722..dc98e21 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -15,30 +15,22 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-jdbc"
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
- "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
- "org.postgresql" % "postgresql" % "9.4-1204-jdbc41",
"org.scalikejdbc" %% "scalikejdbc" % "2.3.5",
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.postgresql" % "postgresql" % "9.4-1204-jdbc41" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
-
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
index 2e6ee83..ff16d5d 100644
--- a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
@@ -15,16 +15,17 @@
* limitations under the License.
*/
-
package org.apache.predictionio.data.storage.jdbc
import java.sql.{DriverManager, ResultSet}
import com.github.nscala_time.time.Imports._
-import org.apache.predictionio.data.storage.{DataMap, Event, PEvents, StorageClientConfig}
+import org.apache.predictionio.data.storage.{
+ DataMap, Event, PEvents, StorageClientConfig}
+import org.apache.predictionio.data.SparkVersionDependent
import org.apache.spark.SparkContext
import org.apache.spark.rdd.{JdbcRDD, RDD}
-import org.apache.spark.sql.{SQLContext, SaveMode}
+import org.apache.spark.sql.SaveMode
import org.json4s.JObject
import org.json4s.native.Serialization
import scalikejdbc._
@@ -32,6 +33,7 @@ import scalikejdbc._
/** JDBC implementation of [[PEvents]] */
class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String) extends PEvents {
@transient private implicit lazy val formats = org.json4s.DefaultFormats
+
def find(
appId: Int,
channelId: Option[Int] = None,
@@ -42,6 +44,7 @@ class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String
eventNames: Option[Seq[String]] = None,
targetEntityType: Option[Option[String]] = None,
targetEntityId: Option[Option[String]] = None)(sc: SparkContext): RDD[Event] = {
+
val lower = startTime.map(_.getMillis).getOrElse(0.toLong)
/** Change the default upper bound from +100 to +1 year because MySQL's
* FROM_UNIXTIME(t) will return NULL if we use +100 years.
@@ -118,13 +121,12 @@ class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String
}
def write(events: RDD[Event], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
- val sqlContext = new SQLContext(sc)
-
- import sqlContext.implicits._
+ val sqlSession = SparkVersionDependent.sqlSession(sc)
+ import sqlSession.implicits._
val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
- val eventTableColumns = Seq[String](
+ val eventsColumnNamesInDF = Seq[String](
"id"
, "event"
, "entityType"
@@ -139,11 +141,16 @@ class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String
, "creationTime"
, "creationTimeZone")
+ // Necessary for handling postgres "case-sensitivity"
+ val eventsColumnNamesInSQL = JDBCUtils.driverType(client) match {
+ case "postgresql" => eventsColumnNamesInDF.map(_.toLowerCase)
+ case _ => eventsColumnNamesInDF
+ }
val eventDF = events.map(x =>
Event(eventId = None, event = x.event, entityType = x.entityType,
entityId = x.entityId, targetEntityType = x.targetEntityType,
targetEntityId = x.targetEntityId, properties = x.properties,
- eventTime = x.eventTime, tags = x.tags, prId= x.prId,
+ eventTime = x.eventTime, tags = x.tags, prId = x.prId,
creationTime = x.eventTime)
)
.map { event =>
@@ -160,9 +167,8 @@ class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String
, event.prId
, new java.sql.Timestamp(event.creationTime.getMillis)
, event.creationTime.getZone.getID)
- }.toDF(eventTableColumns:_*)
+ }.toDF(eventsColumnNamesInSQL:_*)
- // spark version 1.4.0 or higher
val prop = new java.util.Properties
prop.setProperty("user", config.properties("USERNAME"))
prop.setProperty("password", config.properties("PASSWORD"))
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index f9e101d..f8c6b8e 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -15,27 +15,19 @@
* limitations under the License.
*/
+import PIOBuild._
+
name := "apache-predictionio-data-localfs"
libraryDependencies ++= Seq(
"org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
- "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
- "org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.specs2" %% "specs2" % "2.3.13" % "test")
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test")
parallelExecution in Test := false
pomExtra := childrenPomExtra.value
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
-
-assemblyMergeStrategy in assembly := {
- case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
- case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false)
// skip test in assembly
test in assembly := {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index 94f5688..619bdf8 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -17,16 +17,25 @@
FROM predictionio/pio
-ENV SPARK_VERSION 1.6.3
-ENV ELASTICSEARCH_VERSION 5.2.1
-ENV HBASE_VERSION 1.0.0
+ARG SPARK_ARCHIVE
+ARG SPARK_DIR
+ARG PGSQL_JAR
+ARG PIO_SCALA_VERSION
+ARG PIO_SPARK_VERSION
+ARG PIO_ELASTICSEARCH_VERSION
+
+ENV PIO_SCALA_VERSION=$PIO_SCALA_VERSION
+ENV PIO_SPARK_VERSION=$PIO_SPARK_VERSION
+ENV PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION
+
+ENV PGSQL_JAR=$PGSQL_JAR
-ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
# WORKAROUND: es-hadoop stops on RDD#take(1)
-RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
-ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
+ADD docker-files/${SPARK_ARCHIVE} /vendors
+RUN echo "spark.locality.wait.node 0s" > /vendors/${SPARK_DIR}/conf/spark-defaults.conf
+ENV SPARK_HOME /vendors/${SPARK_DIR}
-COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
+COPY docker-files/${PGSQL_JAR} /drivers/${PGSQL_JAR}
COPY docker-files/init.sh init.sh
COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/Dockerfile-es1
----------------------------------------------------------------------
diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1
deleted file mode 100644
index 73d4c36..0000000
--- a/tests/Dockerfile-es1
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM predictionio/pio
-
-ENV SPARK_VERSION 1.6.3
-ENV ELASTICSEARCH_VERSION 1.7
-ENV HBASE_VERSION 1.0.0
-
-ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
-# WORKAROUND: es-hadoop stops on RDD#take(1)
-RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
-ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
-
-COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
-COPY docker-files/init.sh init.sh
-COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
-COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh
-COPY docker-files/pgpass /root/.pgpass
-RUN chmod 600 /root/.pgpass
-
-# Python
-RUN pip install python-dateutil
-RUN pip install pytz
-
-# Default repositories setup
-ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
-ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL
-ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL
-
-# JVM settings
-ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M'
-
-# Expose relevant ports
-# pio engine
-EXPOSE 8000
-# eventserver
-EXPOSE 7070
-
-ENV SLEEP_TIME 30
-
-ENTRYPOINT ["/init.sh"]
-CMD 'bash'
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/README.md
----------------------------------------------------------------------
diff --git a/tests/README.md b/tests/README.md
index 95b3fdf..236d168 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -34,6 +34,12 @@ To download the image run:
$ docker pull predictionio/pio-testing
```
+To build the image use the script:
+```
+$ tests/docker-build.sh <image_name>
+```
+This is necessary to infer proper versions of dependencies e.g. Spark to be included in the image.
+
The most convenient way to make use of it is to execute ***run_docker.sh*** script passing it the configuration, the path to PredictionIO's repository with archived snapshot and the command to run. When no command is provided it opens a bash shell inside the docker image. Example of usage:
```sh
$ ./run_docker.sh ELASTICSEARCH HBASE LOCALFS \
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index 8cbe2c0..e9d5792 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -18,12 +18,10 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $DIR/../conf/pio-vendors.sh
+
# Print a summary of containers used
docker ps -a
# Clean up used containers
-if [ "$ES_VERSION" = "1" ]; then
- docker-compose -f $DIR/docker-compose-es1.yml down
-else
- docker-compose -f $DIR/docker-compose.yml down
-fi
+docker-compose -f $DIR/docker-compose.yml down
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/before_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh
index 9ec7a4a..5889a60 100755
--- a/tests/before_script.travis.sh
+++ b/tests/before_script.travis.sh
@@ -18,4 +18,4 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-$DIR/build-docker.sh
+$DIR/build_docker.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
deleted file mode 100755
index ed43715..0000000
--- a/tests/build-docker.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash -ex
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ ! -f $DIR/docker-files/spark-1.6.3-bin-hadoop2.6.tgz ]; then
- wget http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz
- mv spark-1.6.3-bin-hadoop2.6.tgz $DIR/docker-files/
-fi
-
-if [ ! -f $DIR/docker-files/postgresql-9.4-1204.jdbc41.jar ]; then
- wget https://jdbc.postgresql.org/download/postgresql-9.4-1204.jdbc41.jar
- mv postgresql-9.4-1204.jdbc41.jar $DIR/docker-files/
-fi
-
-docker pull predictionio/pio-testing-base
-pushd $DIR/..
-if [ -z "$ES_VERSION" ]; then
- ./make-distribution.sh
-else
- ./make-distribution.sh --with-es=$ES_VERSION
-fi
-sbt/sbt clean
-mkdir assembly
-cp dist/lib/*.jar assembly/
-mkdir -p lib/spark
-cp dist/lib/spark/*.jar lib/spark
-rm *.tar.gz
-docker build -t predictionio/pio .
-popd
-
-if [ "$ES_VERSION" = "1" ]; then
- docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR
-else
- docker build -t predictionio/pio-testing $DIR
-fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
new file mode 100755
index 0000000..af30cb8
--- /dev/null
+++ b/tests/build_docker.sh
@@ -0,0 +1,57 @@
+#!/bin/bash -ex
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+docker pull predictionio/pio-testing-base
+
+pushd $DIR/..
+
+source conf/pio-vendors.sh
+if [ ! -f $DIR/docker-files/${PGSQL_JAR} ]; then
+ wget $PGSQL_DOWNLOAD
+ mv ${PGSQL_JAR} $DIR/docker-files/
+fi
+if [ ! -f $DIR/docker-files/${SPARK_ARCHIVE} ]; then
+ wget $SPARK_DOWNLOAD
+ mv $SPARK_ARCHIVE $DIR/docker-files/
+fi
+
+./make-distribution.sh \
+ -Dscala.version=$PIO_SCALA_VERSION \
+ -Dspark.version=$PIO_SPARK_VERSION \
+ -Dhadoop.version=$PIO_HADOOP_VERSION \
+ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
+sbt/sbt clean storage/clean
+rm -rf assembly
+mkdir assembly
+cp dist/lib/*.jar assembly/
+rm -rf lib/spark
+mkdir -p lib/spark
+cp dist/lib/spark/*.jar lib/spark
+rm *.tar.gz
+docker build -t predictionio/pio .
+popd
+
+docker build -t predictionio/pio-testing $DIR \
+ --build-arg SPARK_ARCHIVE=$SPARK_ARCHIVE \
+ --build-arg SPARK_DIR=$SPARK_DIR \
+ --build-arg PGSQL_JAR=$PGSQL_JAR \
+ --build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \
+ --build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION \
+ --build-arg PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-compose-es1.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml
deleted file mode 100644
index 381f41c..0000000
--- a/tests/docker-compose-es1.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "2"
-services:
- elasticsearch:
- image: elasticsearch:1
- environment:
- - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- hbase:
- image: harisekhon/hbase:1.0
- postgres:
- image: postgres:9
- environment:
- POSTGRES_USER: pio
- POSTGRES_PASSWORD: pio
- POSTGRES_INITDB_ARGS: --encoding=UTF8
- pio-testing:
- image: predictionio/pio-testing-es1:latest
- depends_on:
- - elasticsearch
- - hbase
- - postgres
- volumes:
- - ~/.ivy2:/root/.ivy2
- - ~/.sbt:/root/.sbt
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 06126a5..ac1d91d 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -16,7 +16,7 @@
version: "2"
services:
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
+ image: ${ES_IMAGE}:${ES_TAG}
environment:
- xpack.security.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/00779c3d/tests/docker-files/env-conf/pio-env-es1.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh
deleted file mode 100644
index e1076ba..0000000
--- a/tests/docker-files/env-conf/pio-env-es1.sh
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copy this file as pio-env.sh and edit it for your site's configuration.
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# PredictionIO Main Configuration
-#
-# This section controls core behavior of PredictionIO. It is very likely that
-# you need to change these to fit your site.
-
-# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$SPARK_HOME
-
-POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
-MYSQL_JDBC_DRIVER=
-
-# ES_CONF_DIR: You must configure this if you have advanced configuration for
-# your Elasticsearch setup.
-# ES_CONF_DIR=/opt/elasticsearch
-
-# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO
-# with Hadoop 2.
-# HADOOP_CONF_DIR=/opt/hadoop
-
-# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO
-# with HBase on a remote cluster.
-HBASE_CONF_DIR=$PIO_HOME/conf
-
-# Filesystem paths where PredictionIO uses as block storage.
-PIO_FS_BASEDIR=$HOME/.pio_store
-PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines
-PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp
-
-# PredictionIO Storage Configuration
-#
-# This section controls programs that make use of PredictionIO's built-in
-# storage facilities. Default values are shown below.
-#
-# For more information on storage configuration please refer to
-# https://docs.prediction.io/system/anotherdatastore/
-
-# Storage Repositories
-
-# Default is to use PostgreSQL
-PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta
-PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
-
-PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event
-PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
-
-PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model
-PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
-
-# Storage Data Sources
-
-# PostgreSQL Default Settings
-# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL
-# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and
-# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly
-PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
-PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio
-PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio
-PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
-
-# MySQL Example
-# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc
-# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio
-# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio
-# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio
-
-# Elasticsearch Example
-PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
-#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
-PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
-PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
-#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
-
-# Local File System Example
-PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
-PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models
-
-# HBase Example
-PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
-#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME
-
-# HDFS config
-PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs
-PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models
[33/50] [abbrv] incubator-predictionio git commit: Code cleanup
Posted by do...@apache.org.
Code cleanup
* Update Unidoc plugin
* Javadoc errors cleanup
* Code cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/68708b47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/68708b47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/68708b47
Branch: refs/heads/livedoc
Commit: 68708b4767278dbcdb1935501031fb759a2b5e85
Parents: a4edda1
Author: shimamoto <ch...@gmail.com>
Authored: Mon Mar 20 16:27:41 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 20 16:27:41 2017 -0700
----------------------------------------------------------------------
build.sbt | 35 +++++++-------
common/build.sbt | 8 ++--
.../controller/IdentityPreparator.scala | 6 +--
data/build.sbt | 4 +-
.../predictionio/data/storage/Storage.scala | 5 +-
.../predictionio/data/view/LBatchView.scala | 48 ++++++--------------
e2/build.sbt | 2 +-
project/unidoc.sbt | 2 +-
storage/elasticsearch/build.sbt | 3 +-
storage/elasticsearch1/build.sbt | 3 +-
storage/hbase/build.sbt | 3 +-
storage/hdfs/build.sbt | 3 +-
storage/jdbc/build.sbt | 3 +-
storage/localfs/build.sbt | 3 +-
tools/build.sbt | 8 ++--
.../org/apache/predictionio/tools/Common.scala | 11 +----
.../apache/predictionio/tools/RunServer.scala | 2 -
.../apache/predictionio/tools/RunWorkflow.scala | 8 +---
.../predictionio/tools/commands/AccessKey.scala | 4 --
.../predictionio/tools/commands/Engine.scala | 7 ---
.../predictionio/tools/commands/Template.scala | 2 -
.../predictionio/tools/console/Console.scala | 14 ------
.../apache/predictionio/tools/console/Pio.scala | 12 +----
.../tools/dashboard/CorsSupport.scala | 1 -
.../tools/dashboard/Dashboard.scala | 4 +-
25 files changed, 63 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index fc47f6a..c0542f7 100644
--- a/build.sbt
+++ b/build.sbt
@@ -16,7 +16,6 @@
*/
import PIOBuild._
-import UnidocKeys._
lazy val scalaSparkDepsVersion = Map(
"2.10" -> Map(
@@ -56,6 +55,8 @@ scalaVersion in ThisBuild := sys.props.getOrElse("scala.version", "2.10.6")
crossScalaVersions in ThisBuild := Seq("2.10.6", "2.11.8")
+crossScalaVersions in ThisBuild := Seq(scalaVersion.value, "2.11.8")
+
scalacOptions in ThisBuild ++= Seq("-deprecation", "-unchecked", "-feature")
scalacOptions in (ThisBuild, Test) ++= Seq("-Yrangepos")
@@ -89,16 +90,12 @@ val pioBuildInfoSettings = buildInfoSettings ++ Seq(
name,
version,
scalaVersion,
+ scalaBinaryVersion,
sbtVersion,
sparkVersion,
hadoopVersion),
buildInfoPackage := "org.apache.predictionio.core")
-// Used temporarily to modify genjavadoc version to "0.10" until unidoc updates it
-val genjavadocSettings: Seq[sbt.Def.Setting[_]] = Seq(
- libraryDependencies += compilerPlugin("com.typesafe.genjavadoc" %% "genjavadoc-plugin" % "0.10" cross CrossVersion.full),
- scalacOptions <+= target map (t => "-P:genjavadoc:out=" + (t / "java")))
-
val conf = file("conf")
val commonSettings = Seq(
@@ -113,44 +110,43 @@ val commonTestSettings = Seq(
val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(publishArtifact := false)
val dataElasticsearch = (project in file("storage/elasticsearch")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
settings(publishArtifact := false)
val dataHbase = (project in file("storage/hbase")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(publishArtifact := false)
val dataHdfs = (project in file("storage/hdfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(publishArtifact := false)
val dataJdbc = (project in file("storage/jdbc")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(publishArtifact := false)
val dataLocalfs = (project in file("storage/localfs")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(publishArtifact := false)
val common = (project in file("common")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
disablePlugins(sbtassembly.AssemblyPlugin)
val data = (project in file("data")).
dependsOn(common).
settings(commonSettings: _*).
settings(commonTestSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(unmanagedSourceDirectories in Compile +=
sourceDirectory.value / s"main/spark-${majorVersion(sparkVersion.value)}").
disablePlugins(sbtassembly.AssemblyPlugin)
@@ -159,7 +155,7 @@ val core = (project in file("core")).
dependsOn(data).
settings(commonSettings: _*).
settings(commonTestSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
settings(pioBuildInfoSettings: _*).
enablePlugins(SbtTwirl).
disablePlugins(sbtassembly.AssemblyPlugin)
@@ -169,13 +165,13 @@ val tools = (project in file("tools")).
dependsOn(data).
settings(commonSettings: _*).
settings(commonTestSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
enablePlugins(SbtTwirl).
settings(publishArtifact := false)
val e2 = (project in file("e2")).
settings(commonSettings: _*).
- settings(genjavadocSettings: _*).
+ enablePlugins(GenJavadocPlugin).
disablePlugins(sbtassembly.AssemblyPlugin)
val dataEs = if (majorVersion(es) == 1) dataElasticsearch1 else dataElasticsearch
@@ -193,9 +189,10 @@ val storage = (project in file("storage"))
val root = (project in file(".")).
settings(commonSettings: _*).
- // settings(scalaJavaUnidocSettings: _*).
- settings(unidocSettings: _*).
+ enablePlugins(ScalaUnidocPlugin).
settings(
+ unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(dataElasticsearch),
+ unidocProjectFilter in (JavaUnidoc, unidoc) := inAnyProject -- inProjects(dataElasticsearch),
scalacOptions in (ScalaUnidoc, unidoc) ++= Seq(
"-groups",
"-skip-packages",
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/common/build.sbt
----------------------------------------------------------------------
diff --git a/common/build.sbt b/common/build.sbt
index 5a0fba1..32afb97 100644
--- a/common/build.sbt
+++ b/common/build.sbt
@@ -20,10 +20,8 @@ import PIOBuild._
name := "apache-predictionio-common"
libraryDependencies ++= Seq(
- "io.spray" %% "spray-can" % "1.3.3",
- "io.spray" %% "spray-routing" % "1.3.3",
- "com.typesafe.akka" %% "akka-actor" % akkaVersion.value,
- "com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value
-)
+ "io.spray" %% "spray-can" % "1.3.3",
+ "io.spray" %% "spray-routing" % "1.3.3",
+ "com.typesafe.akka" %% "akka-actor" % akkaVersion.value)
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala b/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
index bba1b3e..a82f493 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/IdentityPreparator.scala
@@ -55,19 +55,18 @@ object IdentityPreparator {
* @tparam TD Training data class.
* @group Preparator
*/
-@deprecated("Use IdentityPreparator instead.", "0.9.2")
class PIdentityPreparator[TD] extends IdentityPreparator[TD]
/** DEPRECATED. Use [[IdentityPreparator]] instead.
*
* @group Preparator
*/
-@deprecated("Use IdentityPreparator instead.", "0.9.2")
object PIdentityPreparator {
/** Produces an instance of the class of [[IdentityPreparator]].
*
* @param ds Instance of the class of the data source for this preparator.
*/
+ @deprecated("Use IdentityPreparator instead.", "0.9.2")
def apply[TD](ds: Class[_ <: BaseDataSource[TD, _, _, _]]): Class[IdentityPreparator[TD]] =
classOf[IdentityPreparator[TD]]
}
@@ -77,19 +76,18 @@ object PIdentityPreparator {
* @tparam TD Training data class.
* @group Preparator
*/
-@deprecated("Use IdentityPreparator instead.", "0.9.2")
class LIdentityPreparator[TD] extends IdentityPreparator[TD]
/** DEPRECATED. Use [[IdentityPreparator]] instead.
*
* @group Preparator
*/
-@deprecated("Use IdentityPreparator instead.", "0.9.2")
object LIdentityPreparator {
/** Produces an instance of the class of [[IdentityPreparator]].
*
* @param ds Instance of the class of the data source for this preparator.
*/
+ @deprecated("Use IdentityPreparator instead.", "0.9.2")
def apply[TD](ds: Class[_ <: BaseDataSource[TD, _, _, _]]): Class[IdentityPreparator[TD]] =
classOf[IdentityPreparator[TD]]
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index ddd085d..b6b2410 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -21,14 +21,12 @@ name := "apache-predictionio-data"
libraryDependencies ++= Seq(
"com.github.nscala-time" %% "nscala-time" % "2.6.0",
- "com.google.guava" % "guava" % "19.0",
"io.spray" %% "spray-can" % "1.3.3",
"io.spray" %% "spray-routing" % "1.3.3",
+ "io.spray" %% "spray-testkit" % "1.3.3" % "test",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
"org.json4s" %% "json4s-native" % json4sVersion.value,
- "org.scalikejdbc" %% "scalikejdbc" % "2.3.2",
- "io.spray" %% "spray-testkit" % "1.3.3" % "test",
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index 3374f5f..6372446 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -91,7 +91,10 @@ case class StorageClientConfig(
class StorageClientException(message: String, cause: Throwable)
extends RuntimeException(message, cause)
-@deprecated("Use StorageException", "0.9.2")
+/** DEPRECATED. Use [[StorageException]].
+ *
+ * @deprecated Use [[StorageException]]
+ */
private[predictionio] case class StorageError(message: String)
/** :: DeveloperApi ::
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala b/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
index 5345eb0..d9c53ef 100644
--- a/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/view/LBatchView.scala
@@ -28,47 +28,54 @@ import scala.language.implicitConversions
import scala.concurrent.ExecutionContext.Implicits.global // TODO
-@deprecated("Use LEvents or LEventStore instead.", "0.9.2")
object ViewPredicates {
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getStartTimePredicate(startTimeOpt: Option[DateTime])
: (Event => Boolean) = {
startTimeOpt.map(getStartTimePredicate).getOrElse(_ => true)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getStartTimePredicate(startTime: DateTime): (Event => Boolean) = {
e => (!(e.eventTime.isBefore(startTime) || e.eventTime.isEqual(startTime)))
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getUntilTimePredicate(untilTimeOpt: Option[DateTime])
: (Event => Boolean) = {
untilTimeOpt.map(getUntilTimePredicate).getOrElse(_ => true)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getUntilTimePredicate(untilTime: DateTime): (Event => Boolean) = {
_.eventTime.isBefore(untilTime)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getEntityTypePredicate(entityTypeOpt: Option[String]): (Event => Boolean)
= {
entityTypeOpt.map(getEntityTypePredicate).getOrElse(_ => true)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getEntityTypePredicate(entityType: String): (Event => Boolean) = {
(_.entityType == entityType)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getEventPredicate(eventOpt: Option[String]): (Event => Boolean)
= {
eventOpt.map(getEventPredicate).getOrElse(_ => true)
}
+ @deprecated("Use LEvents or LEventStore instead.", "0.9.2")
def getEventPredicate(event: String): (Event => Boolean) = {
(_.event == event)
}
}
-@deprecated("Use LEvents instead.", "0.9.2")
object ViewAggregators {
+ @deprecated("Use LEvents instead.", "0.9.2")
def getDataMapAggregator(): ((Option[DataMap], Event) => Option[DataMap]) = {
(p, e) => {
e.event match {
@@ -93,19 +100,20 @@ object ViewAggregators {
}
}
-@deprecated("Use LEvents instead.", "0.9.2")
object EventSeq {
// Need to
// >>> import scala.language.implicitConversions
// to enable implicit conversion. Only import in the code where this is
// necessary to avoid confusion.
+ @deprecated("Use LEvents instead.", "0.9.2")
implicit def eventSeqToList(es: EventSeq): List[Event] = es.events
+ @deprecated("Use LEvents instead.", "0.9.2")
implicit def listToEventSeq(l: List[Event]): EventSeq = new EventSeq(l)
}
-@deprecated("Use LEvents instead.", "0.9.2")
class EventSeq(val events: List[Event]) {
+ @deprecated("Use LEvents instead.", "0.9.2")
def filter(
eventOpt: Option[String] = None,
entityTypeOpt: Option[String] = None,
@@ -119,8 +127,10 @@ class EventSeq(val events: List[Event]) {
.filter(ViewPredicates.getEntityTypePredicate(entityTypeOpt))
}
+ @deprecated("Use LEvents instead.", "0.9.2")
def filter(p: (Event => Boolean)): EventSeq = events.filter(p)
+ @deprecated("Use LEvents instead.", "0.9.2")
def aggregateByEntityOrdered[T](init: T, op: (T, Event) => T)
: Map[String, T] = {
events
@@ -133,7 +143,6 @@ class EventSeq(val events: List[Event]) {
}
-@deprecated("Use LEventStore instead.", "0.9.2")
class LBatchView(
val appId: Int,
val startTime: Option[DateTime],
@@ -156,6 +165,7 @@ class LBatchView(
* @param untilTimeOpt if specified, only aggregate event until (exclusive)
* endTimeOpt
*/
+ @deprecated("Use LEventStore instead.", "0.9.2")
def aggregateProperties(
entityType: String,
startTimeOpt: Option[DateTime] = None,
@@ -172,32 +182,4 @@ class LBatchView(
.mapValues(_.get)
}
-
- /*
- def aggregateByEntityOrdered[T](
- predicate: Event => Boolean,
- init: T,
- op: (T, Event) => T): Map[String, T] = {
-
- _events
- .filter( predicate(_) )
- .groupBy( _.entityId )
- .mapValues( _.sortBy(_.eventTime.getMillis).foldLeft[T](init)(op))
- .toMap
-
- }
- */
-
- /*
- def groupByEntityOrdered[T](
- predicate: Event => Boolean,
- map: Event => T): Map[String, Seq[T]] = {
-
- _events
- .filter( predicate(_) )
- .groupBy( _.entityId )
- .mapValues( _.sortBy(_.eventTime.getMillis).map(map(_)) )
- .toMap
- }
- */
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/e2/build.sbt
----------------------------------------------------------------------
diff --git a/e2/build.sbt b/e2/build.sbt
index 0774a55..29c1d48 100644
--- a/e2/build.sbt
+++ b/e2/build.sbt
@@ -23,6 +23,6 @@ parallelExecution in Test := false
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-mllib" % sparkVersion.value % "provided",
- "org.scalatest" %% "scalatest" % "2.2.5" % "test")
+ "org.scalatest" %% "scalatest" % "2.2.5" % "test")
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/project/unidoc.sbt
----------------------------------------------------------------------
diff --git a/project/unidoc.sbt b/project/unidoc.sbt
index 7700b32..f9c20f8 100644
--- a/project/unidoc.sbt
+++ b/project/unidoc.sbt
@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.3.3")
+addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/elasticsearch/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch/build.sbt b/storage/elasticsearch/build.sbt
index 52dc965..091b354 100644
--- a/storage/elasticsearch/build.sbt
+++ b/storage/elasticsearch/build.sbt
@@ -43,4 +43,5 @@ assemblyShadeRules in assembly := Seq(
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / s"pio-data-elasticsearch-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/elasticsearch1/build.sbt
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/build.sbt b/storage/elasticsearch1/build.sbt
index bb68b11..6ed4df0 100644
--- a/storage/elasticsearch1/build.sbt
+++ b/storage/elasticsearch1/build.sbt
@@ -33,4 +33,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-elasticsearch1-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / s"pio-data-elasticsearch1-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
index 19aa126..c33db2d 100644
--- a/storage/hbase/build.sbt
+++ b/storage/hbase/build.sbt
@@ -44,4 +44,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / s"pio-data-hbase-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
index 8bae2a8..3fd8d22 100644
--- a/storage/hdfs/build.sbt
+++ b/storage/hdfs/build.sbt
@@ -38,4 +38,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
index dc98e21..3c5026a 100644
--- a/storage/jdbc/build.sbt
+++ b/storage/jdbc/build.sbt
@@ -35,4 +35,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / s"pio-data-jdbc-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
index f8c6b8e..3413e97 100644
--- a/storage/localfs/build.sbt
+++ b/storage/localfs/build.sbt
@@ -32,4 +32,5 @@ assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeSca
// skip test in assembly
test in assembly := {}
-assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile /
+ "assembly" / "spark" / s"pio-data-localfs-assembly-${version.value}.jar"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/build.sbt
----------------------------------------------------------------------
diff --git a/tools/build.sbt b/tools/build.sbt
index 483a591..a39c6b0 100644
--- a/tools/build.sbt
+++ b/tools/build.sbt
@@ -27,7 +27,7 @@ libraryDependencies ++= Seq(
"io.spray" %% "spray-testkit" % "1.3.3" % "test",
"org.specs2" %% "specs2" % "2.3.13" % "test")
-dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.18"
+dependencyOverrides += "org.slf4j" % "slf4j-log4j12" % "1.7.18"
assemblyMergeStrategy in assembly := {
case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
@@ -37,7 +37,7 @@ assemblyMergeStrategy in assembly := {
oldStrategy(x)
}
-excludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
+assemblyExcludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
cp filter { _.data.getName match {
case "reflectasm-1.10.1.jar" => true
case "kryo-3.0.3.jar" => true
@@ -54,8 +54,8 @@ assemblyShadeRules in assembly := Seq(
// skip test in assembly
test in assembly := {}
-outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile /
- "assembly" / ("pio-assembly-" + version.value + ".jar")
+assemblyOutputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile /
+ "assembly" / s"pio-assembly-${version.value}.jar"
cleanFiles <+= baseDirectory { base => base.getParentFile / "assembly" }
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
index 7d04c07..0ddd644 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -65,14 +65,6 @@ object Common extends EitherLogging {
}
}
- def versionNoPatch(fullVersion: String): String = {
- val v = """^(\d+\.\d+)""".r
- val versionNoPatch = for {
- v(np) <- v findFirstIn fullVersion
- } yield np
- versionNoPatch.getOrElse(fullVersion)
- }
-
def getEngineDirPath(directory: Option[String]): String = {
new File(directory.getOrElse(".")).getCanonicalPath
}
@@ -95,9 +87,8 @@ object Common extends EitherLogging {
val engineDir = getEngineDirPath(Some(directory))
val libFiles = jarFilesForScalaFilter(
jarFilesAt(new File(engineDir, "lib")))
- val scalaVersionNoPatch = Common.versionNoPatch(BuildInfo.scalaVersion)
val targetFiles = jarFilesForScalaFilter(jarFilesAt(new File(engineDir,
- "target" + File.separator + s"scala-${scalaVersionNoPatch}")))
+ "target" + File.separator + s"scala-${BuildInfo.scalaBinaryVersion}")))
// Use libFiles is target is empty.
if (targetFiles.size > 0) targetFiles else libFiles
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
index db055b5..0c5b635 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
@@ -20,12 +20,10 @@ package org.apache.predictionio.tools
import org.apache.predictionio.tools.Common._
import org.apache.predictionio.tools.ReturnTypes._
-import org.apache.predictionio.workflow.WorkflowUtils
import org.apache.predictionio.workflow.JsonExtractorOption
import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
import java.io.File
-import java.net.URI
import grizzled.slf4j.Logging
import scala.sys.process._
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index 462770e..6df0750 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -18,20 +18,14 @@
package org.apache.predictionio.tools
import org.apache.predictionio.tools.console.Console
-import org.apache.predictionio.tools.Runner
import org.apache.predictionio.tools.Common._
import org.apache.predictionio.tools.ReturnTypes._
-import org.apache.predictionio.workflow.{WorkflowUtils, JsonExtractorOption}
+import org.apache.predictionio.workflow.JsonExtractorOption
import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
import java.io.File
-import java.net.URI
import grizzled.slf4j.Logging
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
-
import scala.sys.process._
case class WorkflowArgs(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
index 715cf8f..95acc92 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/AccessKey.scala
@@ -18,13 +18,9 @@
package org.apache.predictionio.tools.commands
import org.apache.predictionio.data.storage
-import org.apache.predictionio.tools
import org.apache.predictionio.tools.EitherLogging
import org.apache.predictionio.tools.ReturnTypes._
-import grizzled.slf4j.Logging
-import scala.util.Either
-
object AccessKey extends EitherLogging {
def create(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index c4dc8d4..3924117 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -29,13 +29,6 @@ import org.apache.predictionio.tools.ReturnTypes._
import org.apache.predictionio.workflow.WorkflowUtils
import org.apache.commons.io.FileUtils
-import org.json4s.native.Serialization.read
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-import scala.io.Source
-import scala.util.Random
import scala.collection.JavaConversions._
import scala.sys.process._
import scalaj.http.Http
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
index 1476598..8c6b288 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Template.scala
@@ -21,13 +21,11 @@ import java.io.File
import scala.io.Source
-import grizzled.slf4j.Logging
import org.apache.predictionio.core.BuildInfo
import org.apache.predictionio.tools.EitherLogging
import org.apache.predictionio.tools.ReturnTypes._
import org.json4s._
import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write, read}
import semverfi._
case class TemplateMetaData(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index 810fee0..535905a 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -22,31 +22,17 @@ import java.io.File
import java.net.URI
import grizzled.slf4j.Logging
-import org.apache.commons.io.FileUtils
-import org.apache.predictionio.controller.Utils
import org.apache.predictionio.core.BuildInfo
-import org.apache.predictionio.data.api.{EventServer, EventServerConfig}
-import org.apache.predictionio.data.storage
-import org.apache.predictionio.tools.{RunServer, RunWorkflow, Common}
import org.apache.predictionio.tools.commands.{
DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
BuildArgs, EngineArgs}
import org.apache.predictionio.tools.{
EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
-import org.apache.predictionio.tools.EventServerArgs
-import org.apache.predictionio.tools.admin.{AdminServer, AdminServerConfig}
-import org.apache.predictionio.tools.dashboard.{Dashboard, DashboardConfig}
-import org.apache.predictionio.tools.commands
import org.apache.predictionio.workflow.{JsonExtractorOption, WorkflowUtils}
-import org.apache.predictionio.workflow.JsonExtractorOption.JsonExtractorOption
import org.json4s._
import org.json4s.native.JsonMethods._
-import semverfi._
-import scala.collection.JavaConversions._
import scala.io.Source
-import scala.sys.process._
-import scalaj.http.Http
case class ConsoleArgs(
build: BuildArgs = BuildArgs(),
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
index 19d7590..dd78717 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Pio.scala
@@ -19,25 +19,15 @@ package org.apache.predictionio.tools.console
import org.apache.predictionio.tools.{
EventServerArgs, SparkArgs, WorkflowArgs, ServerArgs, DeployArgs}
-import org.apache.predictionio.tools.commands.Management
import org.apache.predictionio.tools.commands.{
DashboardArgs, AdminServerArgs, ImportArgs, ExportArgs,
- BuildArgs, EngineArgs, AppDescription}
-import org.apache.predictionio.tools.commands.Engine
-import org.apache.predictionio.tools.commands.{
+ BuildArgs, EngineArgs, Management, Engine, Import, Export,
App => AppCmd, AccessKey => AccessKeysCmd}
import org.apache.predictionio.tools.ReturnTypes._
-import org.apache.predictionio.tools.commands.Import
-import org.apache.predictionio.tools.commands.Export
import grizzled.slf4j.Logging
-import scala.concurrent.{Future, ExecutionContext, Await}
-import scala.concurrent.duration._
import scala.language.implicitConversions
import scala.sys.process._
-import java.io.File
-
-import akka.actor.ActorSystem
object Pio extends Logging {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/dashboard/CorsSupport.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/CorsSupport.scala b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/CorsSupport.scala
index 9f7678f..1026996 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/CorsSupport.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/CorsSupport.scala
@@ -22,7 +22,6 @@ package org.apache.predictionio.tools.dashboard
import spray.http.{HttpMethods, HttpMethod, HttpResponse, AllOrigins}
import spray.http.HttpHeaders._
-import spray.http.HttpMethods._
import spray.http.HttpEntity
import spray.routing._
import spray.http.StatusCodes
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/68708b47/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
index 2e9fb26..7d651b1 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/dashboard/Dashboard.scala
@@ -23,8 +23,7 @@ import org.apache.predictionio.authentication.KeyAuthentication
import org.apache.predictionio.configuration.SSLConfiguration
import org.apache.predictionio.data.storage.Storage
import spray.can.server.ServerSettings
-import spray.routing.directives.AuthMagnet
-import scala.concurrent.{Future, ExecutionContext}
+import scala.concurrent.ExecutionContext
import akka.actor.{ActorContext, Actor, ActorSystem, Props}
import akka.io.IO
import akka.pattern.ask
@@ -35,7 +34,6 @@ import spray.can.Http
import spray.http._
import spray.http.MediaTypes._
import spray.routing._
-import spray.routing.authentication.{Authentication, UserPass, BasicAuth}
import scala.concurrent.duration._
[21/50] [abbrv] incubator-predictionio git commit: Disable memory
lock on ES5
Posted by do...@apache.org.
Disable memory lock on ES5
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/947db5e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/947db5e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/947db5e1
Branch: refs/heads/livedoc
Commit: 947db5e1b6e1ff629ef5bcb4412c92309e2a1354
Parents: b6709db
Author: Donald Szeto <do...@apache.org>
Authored: Sun Mar 12 17:36:34 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Mar 12 17:36:34 2017 -0700
----------------------------------------------------------------------
tests/docker-compose.yml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/947db5e1/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index 06126a5..f37ca4a 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -18,6 +18,7 @@ services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
environment:
+ - bootstrap.memory_lock=false
- xpack.security.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
hbase:
[03/50] [abbrv] incubator-predictionio git commit: Clean up unused
files
Posted by do...@apache.org.
Clean up unused files
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/cc5bb916
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/cc5bb916
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/cc5bb916
Branch: refs/heads/livedoc
Commit: cc5bb91659e589b6860e50d41c2107a30b137d3e
Parents: f559222
Author: Donald Szeto <do...@apache.org>
Authored: Sun Feb 26 16:48:30 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Feb 26 16:48:30 2017 -0800
----------------------------------------------------------------------
tests/Dockerfile | 1 -
tests/docker-files/wait-for-postgres.sh | 25 -------------------------
2 files changed, 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc5bb916/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index dff4fa4..d1d048a 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -28,7 +28,6 @@ COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jd
COPY docker-files/init.sh init.sh
COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
COPY docker-files/env-conf/pio-env.sh ${PIO_HOME}/conf/pio-env.sh
-COPY docker-files/wait-for-postgres.sh /wait-for-postgres.sh
COPY docker-files/pgpass /root/.pgpass
RUN chmod 600 /root/.pgpass
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc5bb916/tests/docker-files/wait-for-postgres.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/wait-for-postgres.sh b/tests/docker-files/wait-for-postgres.sh
deleted file mode 100755
index 967e5fc..0000000
--- a/tests/docker-files/wait-for-postgres.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-host="$1"
-
-until psql -h "$host" -U "pio" -c '\l'; do
- >&2 echo "Postgres is unavailable - sleeping"
- sleep 1
-done
[22/50] [abbrv] incubator-predictionio git commit: Working Travis
setup with ES5
Posted by do...@apache.org.
Working Travis setup with ES5
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/0d864657
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/0d864657
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/0d864657
Branch: refs/heads/livedoc
Commit: 0d86465721b4523c44b8fac2f11799666589bac4
Parents: 947db5e
Author: Donald Szeto <do...@apache.org>
Authored: Sun Mar 12 22:57:48 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Mar 12 22:57:48 2017 -0700
----------------------------------------------------------------------
.travis.yml | 3 ++-
tests/docker-compose.yml | 1 -
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d864657/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 799ec3d..9549ac9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -38,7 +38,7 @@ jdk:
services:
- docker
-sudo: false
+sudo: required
cache: false
@@ -57,6 +57,7 @@ before_install:
- sudo mv docker-compose /usr/local/bin
before_script:
+ - sudo sysctl -w vm.max_map_count=262144
- docker-compose -v
- ./tests/before_script.travis.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/0d864657/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index f37ca4a..06126a5 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -18,7 +18,6 @@ services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
environment:
- - bootstrap.memory_lock=false
- xpack.security.enabled=false
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
hbase:
[06/50] [abbrv] incubator-predictionio git commit: Closes #356
Posted by do...@apache.org.
Closes #356
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/b0d37b26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/b0d37b26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/b0d37b26
Branch: refs/heads/livedoc
Commit: b0d37b26f546589088adcecd48169e4f879a2ed2
Parents: 43b0e69
Author: Donald Szeto <do...@apache.org>
Authored: Tue Mar 7 10:23:49 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 10:23:49 2017 -0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[14/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
deleted file mode 100644
index cc07fa4..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase.upgrade
-
-import org.apache.predictionio.annotation.Experimental
-
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventValidation
-import org.apache.predictionio.data.storage.DataMap
-
-import org.apache.hadoop.hbase.client.Scan
-import org.apache.hadoop.hbase.client.HConnection
-import org.apache.hadoop.hbase.client.Result
-import org.apache.hadoop.hbase.TableName
-import org.apache.hadoop.hbase.util.Bytes
-
-import org.joda.time.DateTime
-import org.joda.time.DateTimeZone
-
-import org.json4s.DefaultFormats
-import org.json4s.JObject
-import org.json4s.native.Serialization.{ read, write }
-
-import org.apache.commons.codec.binary.Base64
-
-import scala.collection.JavaConversions._
-
-/** :: Experimental :: */
-@Experimental
-object HB_0_8_0 {
-
- implicit val formats = DefaultFormats
-
- def getByAppId(
- connection: HConnection,
- namespace: String,
- appId: Int): Iterator[Event] = {
- val tableName = TableName.valueOf(namespace, "events")
- val table = connection.getTable(tableName)
- val start = PartialRowKey(appId)
- val stop = PartialRowKey(appId + 1)
- val scan = new Scan(start.toBytes, stop.toBytes)
- val scanner = table.getScanner(scan)
- table.close()
- scanner.iterator().map { resultToEvent(_) }
- }
-
- val colNames: Map[String, Array[Byte]] = Map(
- "event" -> "e",
- "entityType" -> "ety",
- "entityId" -> "eid",
- "targetEntityType" -> "tety",
- "targetEntityId" -> "teid",
- "properties" -> "p",
- "prId" -> "pk", // columna name is 'pk' in 0.8.0/0.8.1
- "eventTimeZone" -> "etz",
- "creationTimeZone" -> "ctz"
- ).mapValues(Bytes.toBytes(_))
-
-
- class RowKey(
- val appId: Int,
- val millis: Long,
- val uuidLow: Long
- ) {
- lazy val toBytes: Array[Byte] = {
- // add UUID least significant bits for multiple actions at the same time
- // (UUID's most significant bits are actually timestamp,
- // use eventTime instead).
- Bytes.toBytes(appId) ++ Bytes.toBytes(millis) ++ Bytes.toBytes(uuidLow)
- }
- override def toString: String = {
- Base64.encodeBase64URLSafeString(toBytes)
- }
- }
-
- object RowKey {
- // get RowKey from string representation
- def apply(s: String): RowKey = {
- try {
- apply(Base64.decodeBase64(s))
- } catch {
- case e: Exception => throw new RowKeyException(
- s"Failed to convert String ${s} to RowKey because ${e}", e)
- }
- }
-
- def apply(b: Array[Byte]): RowKey = {
- if (b.size != 20) {
- val bString = b.mkString(",")
- throw new RowKeyException(
- s"Incorrect byte array size. Bytes: ${bString}.")
- }
-
- new RowKey(
- appId = Bytes.toInt(b.slice(0, 4)),
- millis = Bytes.toLong(b.slice(4, 12)),
- uuidLow = Bytes.toLong(b.slice(12, 20))
- )
- }
- }
-
- class RowKeyException(msg: String, cause: Exception)
- extends Exception(msg, cause) {
- def this(msg: String) = this(msg, null)
- }
-
- case class PartialRowKey(val appId: Int, val millis: Option[Long] = None) {
- val toBytes: Array[Byte] = {
- Bytes.toBytes(appId) ++
- (millis.map(Bytes.toBytes(_)).getOrElse(Array[Byte]()))
- }
- }
-
- def resultToEvent(result: Result): Event = {
- val rowKey = RowKey(result.getRow())
-
- val eBytes = Bytes.toBytes("e")
- // val e = result.getFamilyMap(eBytes)
-
- def getStringCol(col: String): String = {
- val r = result.getValue(eBytes, colNames(col))
- require(r != null,
- s"Failed to get value for column ${col}. " +
- s"Rowkey: ${rowKey.toString} " +
- s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
-
- Bytes.toString(r)
- }
-
- def getOptStringCol(col: String): Option[String] = {
- val r = result.getValue(eBytes, colNames(col))
- if (r == null) {
- None
- } else {
- Some(Bytes.toString(r))
- }
- }
-
- def getTimestamp(col: String): Long = {
- result.getColumnLatestCell(eBytes, colNames(col)).getTimestamp()
- }
-
- val event = getStringCol("event")
- val entityType = getStringCol("entityType")
- val entityId = getStringCol("entityId")
- val targetEntityType = getOptStringCol("targetEntityType")
- val targetEntityId = getOptStringCol("targetEntityId")
- val properties: DataMap = getOptStringCol("properties")
- .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
- val prId = getOptStringCol("prId")
- val eventTimeZone = getOptStringCol("eventTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
- val creationTimeZone = getOptStringCol("creationTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
-
- val creationTime: DateTime = new DateTime(
- getTimestamp("event"), creationTimeZone
- )
-
- Event(
- eventId = Some(RowKey(result.getRow()).toString),
- event = event,
- entityType = entityType,
- entityId = entityId,
- targetEntityType = targetEntityType,
- targetEntityId = targetEntityId,
- properties = properties,
- eventTime = new DateTime(rowKey.millis, eventTimeZone),
- tags = Seq(),
- prId = prId,
- creationTime = creationTime
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
deleted file mode 100644
index 1759561..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase.upgrade
-
-import org.apache.predictionio.annotation.Experimental
-
-import org.apache.predictionio.data.storage.Storage
-import org.apache.predictionio.data.storage.hbase.HBLEvents
-import org.apache.predictionio.data.storage.hbase.HBEventsUtil
-
-import scala.collection.JavaConversions._
-
-/** :: Experimental :: */
-@Experimental
-object Upgrade {
-
- def main(args: Array[String]) {
- val fromAppId = args(0).toInt
- val toAppId = args(1).toInt
- val batchSize = args.lift(2).map(_.toInt).getOrElse(100)
- val fromNamespace = args.lift(3).getOrElse("predictionio_eventdata")
-
- upgrade(fromAppId, toAppId, batchSize, fromNamespace)
- }
-
- /* For upgrade from 0.8.0 or 0.8.1 to 0.8.2 only */
- def upgrade(
- fromAppId: Int,
- toAppId: Int,
- batchSize: Int,
- fromNamespace: String) {
-
- val events = Storage.getLEvents().asInstanceOf[HBLEvents]
-
- // Assume already run "pio app new <newapp>" (new app already created)
- // TODO: check if new table empty and warn user if not
- val newTable = events.getTable(toAppId)
-
- val newTableName = newTable.getName().getNameAsString()
- println(s"Copying data from ${fromNamespace}:events for app ID ${fromAppId}"
- + s" to new HBase table ${newTableName}...")
-
- HB_0_8_0.getByAppId(
- events.client.connection,
- fromNamespace,
- fromAppId).grouped(batchSize).foreach { eventGroup =>
- val puts = eventGroup.map{ e =>
- val (put, rowkey) = HBEventsUtil.eventToPut(e, toAppId)
- put
- }
- newTable.put(puts.toList)
- }
-
- newTable.flushCommits()
- newTable.close()
- println("Done.")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
deleted file mode 100644
index de74d46..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase.upgrade
-
-import org.apache.predictionio.annotation.Experimental
-
-import grizzled.slf4j.Logger
-import org.apache.predictionio.data.storage.Storage
-import org.apache.predictionio.data.storage.DataMap
-import org.apache.predictionio.data.storage.hbase.HBLEvents
-import org.apache.predictionio.data.storage.hbase.HBEventsUtil
-
-import scala.collection.JavaConversions._
-
-import scala.concurrent._
-import ExecutionContext.Implicits.global
-import org.apache.predictionio.data.storage.LEvents
-import scala.concurrent.Await
-import scala.concurrent.duration.Duration
-import java.lang.Thread
-
-object CheckDistribution {
- def entityType(eventClient: LEvents, appId: Int)
- : Map[(String, Option[String]), Int] = {
- eventClient
- .find(appId = appId)
- .foldLeft(Map[(String, Option[String]), Int]().withDefaultValue(0)) {
- case (m, e) => {
- val k = (e.entityType, e.targetEntityType)
- m.updated(k, m(k) + 1)
- }
- }
- }
-
- def runMain(appId: Int) {
- val eventClient = Storage.getLEvents().asInstanceOf[HBLEvents]
-
- entityType(eventClient, appId)
- .toSeq
- .sortBy(-_._2)
- .foreach { println }
-
- }
-
- def main(args: Array[String]) {
- runMain(args(0).toInt)
- }
-
-}
-
-/** :: Experimental :: */
-@Experimental
-object Upgrade_0_8_3 {
- val NameMap = Map(
- "pio_user" -> "user",
- "pio_item" -> "item")
- val RevNameMap = NameMap.toSeq.map(_.swap).toMap
-
- val logger = Logger[this.type]
-
- def main(args: Array[String]) {
- val fromAppId = args(0).toInt
- val toAppId = args(1).toInt
-
- runMain(fromAppId, toAppId)
- }
-
- def runMain(fromAppId: Int, toAppId: Int): Unit = {
- upgrade(fromAppId, toAppId)
- }
-
-
- val obsEntityTypes = Set("pio_user", "pio_item")
- val obsProperties = Set(
- "pio_itypes", "pio_starttime", "pio_endtime",
- "pio_inactive", "pio_price", "pio_rating")
-
- def hasPIOPrefix(eventClient: LEvents, appId: Int): Boolean = {
- eventClient.find(appId = appId).filter( e =>
- (obsEntityTypes.contains(e.entityType) ||
- e.targetEntityType.map(obsEntityTypes.contains(_)).getOrElse(false) ||
- (!e.properties.keySet.forall(!obsProperties.contains(_)))
- )
- ).hasNext
- }
-
- def isEmpty(eventClient: LEvents, appId: Int): Boolean =
- !eventClient.find(appId = appId).hasNext
-
-
- def upgradeCopy(eventClient: LEvents, fromAppId: Int, toAppId: Int) {
- val fromDist = CheckDistribution.entityType(eventClient, fromAppId)
-
- logger.info("FromAppId Distribution")
- fromDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
-
- val events = eventClient
- .find(appId = fromAppId)
- .zipWithIndex
- .foreach { case (fromEvent, index) => {
- if (index % 50000 == 0) {
- // logger.info(s"Progress: $fromEvent $index")
- logger.info(s"Progress: $index")
- }
-
-
- val fromEntityType = fromEvent.entityType
- val toEntityType = NameMap.getOrElse(fromEntityType, fromEntityType)
-
- val fromTargetEntityType = fromEvent.targetEntityType
- val toTargetEntityType = fromTargetEntityType
- .map { et => NameMap.getOrElse(et, et) }
-
- val toProperties = DataMap(fromEvent.properties.fields.map {
- case (k, v) =>
- val newK = if (obsProperties.contains(k)) {
- val nK = k.stripPrefix("pio_")
- logger.info(s"property ${k} will be renamed to ${nK}")
- nK
- } else k
- (newK, v)
- })
-
- val toEvent = fromEvent.copy(
- entityType = toEntityType,
- targetEntityType = toTargetEntityType,
- properties = toProperties)
-
- eventClient.insert(toEvent, toAppId)
- }}
-
-
- val toDist = CheckDistribution.entityType(eventClient, toAppId)
-
- logger.info("Recap fromAppId Distribution")
- fromDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
-
- logger.info("ToAppId Distribution")
- toDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
-
- val fromGood = fromDist
- .toSeq
- .forall { case (k, c) => {
- val (et, tet) = k
- val net = NameMap.getOrElse(et, et)
- val ntet = tet.map(tet => NameMap.getOrElse(tet, tet))
- val nk = (net, ntet)
- val nc = toDist.getOrElse(nk, -1)
- val checkMatch = (c == nc)
- if (!checkMatch) {
- logger.info(s"${k} doesn't match: old has ${c}. new has ${nc}.")
- }
- checkMatch
- }}
-
- val toGood = toDist
- .toSeq
- .forall { case (k, c) => {
- val (et, tet) = k
- val oet = RevNameMap.getOrElse(et, et)
- val otet = tet.map(tet => RevNameMap.getOrElse(tet, tet))
- val ok = (oet, otet)
- val oc = fromDist.getOrElse(ok, -1)
- val checkMatch = (c == oc)
- if (!checkMatch) {
- logger.info(s"${k} doesn't match: new has ${c}. old has ${oc}.")
- }
- checkMatch
- }}
-
- if (!fromGood || !toGood) {
- logger.error("Doesn't match!! There is an import error.")
- } else {
- logger.info("Count matches. Looks like we are good to go.")
- }
- }
-
- /* For upgrade from 0.8.2 to 0.8.3 only */
- def upgrade(fromAppId: Int, toAppId: Int) {
-
- val eventClient = Storage.getLEvents().asInstanceOf[HBLEvents]
-
- require(fromAppId != toAppId,
- s"FromAppId: $fromAppId must be different from toAppId: $toAppId")
-
- if (hasPIOPrefix(eventClient, fromAppId)) {
- require(
- isEmpty(eventClient, toAppId),
- s"Target appId: $toAppId is not empty. Please run " +
- "`pio app data-delete <app_name>` to clean the data before upgrading")
-
- logger.info(s"$fromAppId isEmpty: " + isEmpty(eventClient, fromAppId))
-
- upgradeCopy(eventClient, fromAppId, toAppId)
-
- } else {
- logger.info(s"From appId: ${fromAppId} doesn't contain"
- + s" obsolete entityTypes ${obsEntityTypes} or"
- + s" obsolete properties ${obsProperties}."
- + " No need data migration."
- + s" You can continue to use appId ${fromAppId}.")
- }
-
- logger.info("Done.")
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
deleted file mode 100644
index 08dfb01..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hdfs
-
-import java.io.IOException
-
-import com.google.common.io.ByteStreams
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Model
-import org.apache.predictionio.data.storage.Models
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
-
-class HDFSModels(fs: FileSystem, config: StorageClientConfig, prefix: String)
- extends Models with Logging {
-
- def insert(i: Model): Unit = {
- try {
- val fsdos = fs.create(new Path(s"$prefix${i.id}"))
- fsdos.write(i.models)
- fsdos.close
- } catch {
- case e: IOException => error(e.getMessage)
- }
- }
-
- def get(id: String): Option[Model] = {
- try {
- val p = new Path(s"$prefix$id")
- Some(Model(
- id = id,
- models = ByteStreams.toByteArray(fs.open(p))))
- } catch {
- case e: Throwable =>
- error(e.getMessage)
- None
- }
- }
-
- def delete(id: String): Unit = {
- val p = new Path(s"$prefix$id")
- if (!fs.delete(p, false)) {
- error(s"Unable to delete ${fs.makeQualified(p).toString}!")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
deleted file mode 100644
index bc57f2a..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hdfs
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.BaseStorageClient
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.Path
-
-class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
- with Logging {
- override val prefix = "HDFS"
- val conf = new Configuration
- val fs = FileSystem.get(conf)
- fs.setWorkingDirectory(
- new Path(config.properties.getOrElse("PATH", config.properties("HOSTS"))))
- val client = fs
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
deleted file mode 100644
index a927d78..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-/** HDFS implementation of storage traits, supporting model data only
- *
- * @group Implementation
- */
-package object hdfs {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
deleted file mode 100644
index 437f8ae..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.AccessKey
-import org.apache.predictionio.data.storage.AccessKeys
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-import scala.util.Random
-
-/** JDBC implementation of [[AccessKeys]] */
-class JDBCAccessKeys(client: String, config: StorageClientConfig, prefix: String)
- extends AccessKeys with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "accesskeys")
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- accesskey varchar(64) not null primary key,
- appid integer not null,
- events text)""".execute().apply()
- }
-
- def insert(accessKey: AccessKey): Option[String] = DB localTx { implicit s =>
- val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
- val events = if (accessKey.events.isEmpty) None else Some(accessKey.events.mkString(","))
- sql"""
- insert into $tableName values(
- $key,
- ${accessKey.appid},
- $events)""".update().apply()
- Some(key)
- }
-
- def get(key: String): Option[AccessKey] = DB readOnly { implicit session =>
- sql"SELECT accesskey, appid, events FROM $tableName WHERE accesskey = $key".
- map(resultToAccessKey).single().apply()
- }
-
- def getAll(): Seq[AccessKey] = DB readOnly { implicit session =>
- sql"SELECT accesskey, appid, events FROM $tableName".map(resultToAccessKey).list().apply()
- }
-
- def getByAppid(appid: Int): Seq[AccessKey] = DB readOnly { implicit session =>
- sql"SELECT accesskey, appid, events FROM $tableName WHERE appid = $appid".
- map(resultToAccessKey).list().apply()
- }
-
- def update(accessKey: AccessKey): Unit = DB localTx { implicit session =>
- val events = if (accessKey.events.isEmpty) None else Some(accessKey.events.mkString(","))
- sql"""
- UPDATE $tableName SET
- appid = ${accessKey.appid},
- events = $events
- WHERE accesskey = ${accessKey.key}""".update().apply()
- }
-
- def delete(key: String): Unit = DB localTx { implicit session =>
- sql"DELETE FROM $tableName WHERE accesskey = $key".update().apply()
- }
-
- /** Convert JDBC results to [[AccessKey]] */
- def resultToAccessKey(rs: WrappedResultSet): AccessKey = {
- AccessKey(
- key = rs.string("accesskey"),
- appid = rs.int("appid"),
- events = rs.stringOpt("events").map(_.split(",").toSeq).getOrElse(Nil))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
deleted file mode 100644
index 17e6410..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.App
-import org.apache.predictionio.data.storage.Apps
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-/** JDBC implementation of [[Apps]] */
-class JDBCApps(client: String, config: StorageClientConfig, prefix: String)
- extends Apps with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "apps")
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- id serial not null primary key,
- name text not null,
- description text)""".execute.apply()
- }
-
- def insert(app: App): Option[Int] = DB localTx { implicit session =>
- val q = if (app.id == 0) {
- sql"""
- insert into $tableName (name, description) values(${app.name}, ${app.description})
- """
- } else {
- sql"""
- insert into $tableName values(${app.id}, ${app.name}, ${app.description})
- """
- }
- Some(q.updateAndReturnGeneratedKey().apply().toInt)
- }
-
- def get(id: Int): Option[App] = DB readOnly { implicit session =>
- sql"SELECT id, name, description FROM $tableName WHERE id = ${id}".map(rs =>
- App(
- id = rs.int("id"),
- name = rs.string("name"),
- description = rs.stringOpt("description"))
- ).single().apply()
- }
-
- def getByName(name: String): Option[App] = DB readOnly { implicit session =>
- sql"SELECT id, name, description FROM $tableName WHERE name = ${name}".map(rs =>
- App(
- id = rs.int("id"),
- name = rs.string("name"),
- description = rs.stringOpt("description"))
- ).single().apply()
- }
-
- def getAll(): Seq[App] = DB readOnly { implicit session =>
- sql"SELECT id, name, description FROM $tableName".map(rs =>
- App(
- id = rs.int("id"),
- name = rs.string("name"),
- description = rs.stringOpt("description"))
- ).list().apply()
- }
-
- def update(app: App): Unit = DB localTx { implicit session =>
- sql"""
- update $tableName set name = ${app.name}, description = ${app.description}
- where id = ${app.id}""".update().apply()
- }
-
- def delete(id: Int): Unit = DB localTx { implicit session =>
- sql"DELETE FROM $tableName WHERE id = $id".update().apply()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
deleted file mode 100644
index c9aaca5..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Channel
-import org.apache.predictionio.data.storage.Channels
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-/** JDBC implementation of [[Channels]] */
-class JDBCChannels(client: String, config: StorageClientConfig, prefix: String)
- extends Channels with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "channels")
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- id serial not null primary key,
- name text not null,
- appid integer not null)""".execute().apply()
- }
-
- def insert(channel: Channel): Option[Int] = DB localTx { implicit session =>
- val q = if (channel.id == 0) {
- sql"INSERT INTO $tableName (name, appid) VALUES(${channel.name}, ${channel.appid})"
- } else {
- sql"INSERT INTO $tableName VALUES(${channel.id}, ${channel.name}, ${channel.appid})"
- }
- Some(q.updateAndReturnGeneratedKey().apply().toInt)
- }
-
- def get(id: Int): Option[Channel] = DB localTx { implicit session =>
- sql"SELECT id, name, appid FROM $tableName WHERE id = $id".
- map(resultToChannel).single().apply()
- }
-
- def getByAppid(appid: Int): Seq[Channel] = DB localTx { implicit session =>
- sql"SELECT id, name, appid FROM $tableName WHERE appid = $appid".
- map(resultToChannel).list().apply()
- }
-
- def delete(id: Int): Unit = DB localTx { implicit session =>
- sql"DELETE FROM $tableName WHERE id = $id".update().apply()
- }
-
- def resultToChannel(rs: WrappedResultSet): Channel = {
- Channel(
- id = rs.int("id"),
- name = rs.string("name"),
- appid = rs.int("appid"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
deleted file mode 100644
index 13c374d..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.EngineInstance
-import org.apache.predictionio.data.storage.EngineInstances
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-/** JDBC implementation of [[EngineInstances]] */
-class JDBCEngineInstances(client: String, config: StorageClientConfig, prefix: String)
- extends EngineInstances with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "engineinstances")
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- id varchar(100) not null primary key,
- status text not null,
- startTime timestamp DEFAULT CURRENT_TIMESTAMP,
- endTime timestamp DEFAULT CURRENT_TIMESTAMP,
- engineId text not null,
- engineVersion text not null,
- engineVariant text not null,
- engineFactory text not null,
- batch text not null,
- env text not null,
- sparkConf text not null,
- datasourceParams text not null,
- preparatorParams text not null,
- algorithmsParams text not null,
- servingParams text not null)""".execute().apply()
- }
-
- def insert(i: EngineInstance): String = DB localTx { implicit session =>
- val id = java.util.UUID.randomUUID().toString
- sql"""
- INSERT INTO $tableName VALUES(
- $id,
- ${i.status},
- ${i.startTime},
- ${i.endTime},
- ${i.engineId},
- ${i.engineVersion},
- ${i.engineVariant},
- ${i.engineFactory},
- ${i.batch},
- ${JDBCUtils.mapToString(i.env)},
- ${JDBCUtils.mapToString(i.sparkConf)},
- ${i.dataSourceParams},
- ${i.preparatorParams},
- ${i.algorithmsParams},
- ${i.servingParams})""".update().apply()
- id
- }
-
- def get(id: String): Option[EngineInstance] = DB localTx { implicit session =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- engineId,
- engineVersion,
- engineVariant,
- engineFactory,
- batch,
- env,
- sparkConf,
- datasourceParams,
- preparatorParams,
- algorithmsParams,
- servingParams
- FROM $tableName WHERE id = $id""".map(resultToEngineInstance).
- single().apply()
- }
-
- def getAll(): Seq[EngineInstance] = DB localTx { implicit session =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- engineId,
- engineVersion,
- engineVariant,
- engineFactory,
- batch,
- env,
- sparkConf,
- datasourceParams,
- preparatorParams,
- algorithmsParams,
- servingParams
- FROM $tableName""".map(resultToEngineInstance).list().apply()
- }
-
- def getLatestCompleted(
- engineId: String,
- engineVersion: String,
- engineVariant: String): Option[EngineInstance] =
- getCompleted(engineId, engineVersion, engineVariant).headOption
-
- def getCompleted(
- engineId: String,
- engineVersion: String,
- engineVariant: String): Seq[EngineInstance] = DB localTx { implicit s =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- engineId,
- engineVersion,
- engineVariant,
- engineFactory,
- batch,
- env,
- sparkConf,
- datasourceParams,
- preparatorParams,
- algorithmsParams,
- servingParams
- FROM $tableName
- WHERE
- status = 'COMPLETED' AND
- engineId = $engineId AND
- engineVersion = $engineVersion AND
- engineVariant = $engineVariant
- ORDER BY startTime DESC""".
- map(resultToEngineInstance).list().apply()
- }
-
- def update(i: EngineInstance): Unit = DB localTx { implicit session =>
- sql"""
- update $tableName set
- status = ${i.status},
- startTime = ${i.startTime},
- endTime = ${i.endTime},
- engineId = ${i.engineId},
- engineVersion = ${i.engineVersion},
- engineVariant = ${i.engineVariant},
- engineFactory = ${i.engineFactory},
- batch = ${i.batch},
- env = ${JDBCUtils.mapToString(i.env)},
- sparkConf = ${JDBCUtils.mapToString(i.sparkConf)},
- datasourceParams = ${i.dataSourceParams},
- preparatorParams = ${i.preparatorParams},
- algorithmsParams = ${i.algorithmsParams},
- servingParams = ${i.servingParams}
- where id = ${i.id}""".update().apply()
- }
-
- def delete(id: String): Unit = DB localTx { implicit session =>
- sql"DELETE FROM $tableName WHERE id = $id".update().apply()
- }
-
- /** Convert JDBC results to [[EngineInstance]] */
- def resultToEngineInstance(rs: WrappedResultSet): EngineInstance = {
- EngineInstance(
- id = rs.string("id"),
- status = rs.string("status"),
- startTime = rs.jodaDateTime("startTime"),
- endTime = rs.jodaDateTime("endTime"),
- engineId = rs.string("engineId"),
- engineVersion = rs.string("engineVersion"),
- engineVariant = rs.string("engineVariant"),
- engineFactory = rs.string("engineFactory"),
- batch = rs.string("batch"),
- env = JDBCUtils.stringToMap(rs.string("env")),
- sparkConf = JDBCUtils.stringToMap(rs.string("sparkConf")),
- dataSourceParams = rs.string("datasourceParams"),
- preparatorParams = rs.string("preparatorParams"),
- algorithmsParams = rs.string("algorithmsParams"),
- servingParams = rs.string("servingParams"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
deleted file mode 100644
index 90eb5f3..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.EvaluationInstance
-import org.apache.predictionio.data.storage.EvaluationInstances
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-/** JDBC implementations of [[EvaluationInstances]] */
-class JDBCEvaluationInstances(client: String, config: StorageClientConfig, prefix: String)
- extends EvaluationInstances with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "evaluationinstances")
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- id varchar(100) not null primary key,
- status text not null,
- startTime timestamp DEFAULT CURRENT_TIMESTAMP,
- endTime timestamp DEFAULT CURRENT_TIMESTAMP,
- evaluationClass text not null,
- engineParamsGeneratorClass text not null,
- batch text not null,
- env text not null,
- sparkConf text not null,
- evaluatorResults text not null,
- evaluatorResultsHTML text not null,
- evaluatorResultsJSON text)""".execute().apply()
- }
-
- def insert(i: EvaluationInstance): String = DB localTx { implicit session =>
- val id = java.util.UUID.randomUUID().toString
- sql"""
- INSERT INTO $tableName VALUES(
- $id,
- ${i.status},
- ${i.startTime},
- ${i.endTime},
- ${i.evaluationClass},
- ${i.engineParamsGeneratorClass},
- ${i.batch},
- ${JDBCUtils.mapToString(i.env)},
- ${JDBCUtils.mapToString(i.sparkConf)},
- ${i.evaluatorResults},
- ${i.evaluatorResultsHTML},
- ${i.evaluatorResultsJSON})""".update().apply()
- id
- }
-
- def get(id: String): Option[EvaluationInstance] = DB localTx { implicit session =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- evaluationClass,
- engineParamsGeneratorClass,
- batch,
- env,
- sparkConf,
- evaluatorResults,
- evaluatorResultsHTML,
- evaluatorResultsJSON
- FROM $tableName WHERE id = $id
- """.map(resultToEvaluationInstance).single().apply()
- }
-
- def getAll(): Seq[EvaluationInstance] = DB localTx { implicit session =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- evaluationClass,
- engineParamsGeneratorClass,
- batch,
- env,
- sparkConf,
- evaluatorResults,
- evaluatorResultsHTML,
- evaluatorResultsJSON
- FROM $tableName
- """.map(resultToEvaluationInstance).list().apply()
- }
-
- def getCompleted(): Seq[EvaluationInstance] = DB localTx { implicit s =>
- sql"""
- SELECT
- id,
- status,
- startTime,
- endTime,
- evaluationClass,
- engineParamsGeneratorClass,
- batch,
- env,
- sparkConf,
- evaluatorResults,
- evaluatorResultsHTML,
- evaluatorResultsJSON
- FROM $tableName
- WHERE
- status = 'EVALCOMPLETED'
- ORDER BY starttime DESC
- """.map(resultToEvaluationInstance).list().apply()
- }
-
- def update(i: EvaluationInstance): Unit = DB localTx { implicit session =>
- sql"""
- update $tableName set
- status = ${i.status},
- startTime = ${i.startTime},
- endTime = ${i.endTime},
- evaluationClass = ${i.evaluationClass},
- engineParamsGeneratorClass = ${i.engineParamsGeneratorClass},
- batch = ${i.batch},
- env = ${JDBCUtils.mapToString(i.env)},
- sparkConf = ${JDBCUtils.mapToString(i.sparkConf)},
- evaluatorResults = ${i.evaluatorResults},
- evaluatorResultsHTML = ${i.evaluatorResultsHTML},
- evaluatorResultsJSON = ${i.evaluatorResultsJSON}
- where id = ${i.id}""".update().apply()
- }
-
- def delete(id: String): Unit = DB localTx { implicit session =>
- sql"DELETE FROM $tableName WHERE id = $id".update().apply()
- }
-
- /** Convert JDBC results to [[EvaluationInstance]] */
- def resultToEvaluationInstance(rs: WrappedResultSet): EvaluationInstance = {
- EvaluationInstance(
- id = rs.string("id"),
- status = rs.string("status"),
- startTime = rs.jodaDateTime("startTime"),
- endTime = rs.jodaDateTime("endTime"),
- evaluationClass = rs.string("evaluationClass"),
- engineParamsGeneratorClass = rs.string("engineParamsGeneratorClass"),
- batch = rs.string("batch"),
- env = JDBCUtils.stringToMap(rs.string("env")),
- sparkConf = JDBCUtils.stringToMap(rs.string("sparkConf")),
- evaluatorResults = rs.string("evaluatorResults"),
- evaluatorResultsHTML = rs.string("evaluatorResultsHTML"),
- evaluatorResultsJSON = rs.string("evaluatorResultsJSON"))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
deleted file mode 100644
index dddef67..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.DataMap
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.LEvents
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.joda.time.DateTime
-import org.joda.time.DateTimeZone
-import org.json4s.JObject
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-import scalikejdbc._
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-
-/** JDBC implementation of [[LEvents]] */
-class JDBCLEvents(
- client: String,
- config: StorageClientConfig,
- namespace: String) extends LEvents with Logging {
- implicit private val formats = org.json4s.DefaultFormats
-
- def init(appId: Int, channelId: Option[Int] = None): Boolean = {
-
- // To use index, it must be varchar less than 255 characters on a VARCHAR column
- val useIndex = config.properties.contains("INDEX") &&
- config.properties("INDEX").equalsIgnoreCase("enabled")
-
- val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
- val entityIdIndexName = s"idx_${tableName}_ei"
- val entityTypeIndexName = s"idx_${tableName}_et"
- DB autoCommit { implicit session =>
- if (useIndex) {
- SQL(s"""
- create table if not exists $tableName (
- id varchar(32) not null primary key,
- event varchar(255) not null,
- entityType varchar(255) not null,
- entityId varchar(255) not null,
- targetEntityType text,
- targetEntityId text,
- properties text,
- eventTime timestamp DEFAULT CURRENT_TIMESTAMP,
- eventTimeZone varchar(50) not null,
- tags text,
- prId text,
- creationTime timestamp DEFAULT CURRENT_TIMESTAMP,
- creationTimeZone varchar(50) not null)""").execute().apply()
-
- // create index
- SQL(s"create index $entityIdIndexName on $tableName (entityId)").execute().apply()
- SQL(s"create index $entityTypeIndexName on $tableName (entityType)").execute().apply()
- } else {
- SQL(s"""
- create table if not exists $tableName (
- id varchar(32) not null primary key,
- event text not null,
- entityType text not null,
- entityId text not null,
- targetEntityType text,
- targetEntityId text,
- properties text,
- eventTime timestamp DEFAULT CURRENT_TIMESTAMP,
- eventTimeZone varchar(50) not null,
- tags text,
- prId text,
- creationTime timestamp DEFAULT CURRENT_TIMESTAMP,
- creationTimeZone varchar(50) not null)""").execute().apply()
- }
- true
- }
- }
-
- def remove(appId: Int, channelId: Option[Int] = None): Boolean =
- DB autoCommit { implicit session =>
- SQL(s"""
- drop table ${JDBCUtils.eventTableName(namespace, appId, channelId)}
- """).execute().apply()
- true
- }
-
- def close(): Unit = ConnectionPool.closeAll()
-
- def futureInsert(event: Event, appId: Int, channelId: Option[Int])(
- implicit ec: ExecutionContext): Future[String] = Future {
- DB localTx { implicit session =>
- val id = event.eventId.getOrElse(JDBCUtils.generateId)
- val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
- sql"""
- insert into $tableName values(
- $id,
- ${event.event},
- ${event.entityType},
- ${event.entityId},
- ${event.targetEntityType},
- ${event.targetEntityId},
- ${write(event.properties.toJObject)},
- ${event.eventTime},
- ${event.eventTime.getZone.getID},
- ${if (event.tags.nonEmpty) Some(event.tags.mkString(",")) else None},
- ${event.prId},
- ${event.creationTime},
- ${event.creationTime.getZone.getID}
- )
- """.update().apply()
- id
- }
- }
-
- def futureGet(eventId: String, appId: Int, channelId: Option[Int])(
- implicit ec: ExecutionContext): Future[Option[Event]] = Future {
- DB readOnly { implicit session =>
- val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
- sql"""
- select
- id,
- event,
- entityType,
- entityId,
- targetEntityType,
- targetEntityId,
- properties,
- eventTime,
- eventTimeZone,
- tags,
- prId,
- creationTime,
- creationTimeZone
- from $tableName
- where id = $eventId
- """.map(resultToEvent).single().apply()
- }
- }
-
- def futureDelete(eventId: String, appId: Int, channelId: Option[Int])(
- implicit ec: ExecutionContext): Future[Boolean] = Future {
- DB localTx { implicit session =>
- val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
- sql"""
- delete from $tableName where id = $eventId
- """.update().apply()
- true
- }
- }
-
- def futureFind(
- appId: Int,
- channelId: Option[Int] = None,
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None,
- entityType: Option[String] = None,
- entityId: Option[String] = None,
- eventNames: Option[Seq[String]] = None,
- targetEntityType: Option[Option[String]] = None,
- targetEntityId: Option[Option[String]] = None,
- limit: Option[Int] = None,
- reversed: Option[Boolean] = None
- )(implicit ec: ExecutionContext): Future[Iterator[Event]] = Future {
- DB readOnly { implicit session =>
- val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
- val whereClause = sqls.toAndConditionOpt(
- startTime.map(x => sqls"eventTime >= $x"),
- untilTime.map(x => sqls"eventTime < $x"),
- entityType.map(x => sqls"entityType = $x"),
- entityId.map(x => sqls"entityId = $x"),
- eventNames.map(x =>
- sqls.toOrConditionOpt(x.map(y =>
- Some(sqls"event = $y")
- ): _*)
- ).getOrElse(None),
- targetEntityType.map(x => x.map(y => sqls"targetEntityType = $y")
- .getOrElse(sqls"targetEntityType IS NULL")),
- targetEntityId.map(x => x.map(y => sqls"targetEntityId = $y")
- .getOrElse(sqls"targetEntityId IS NULL"))
- ).map(sqls.where(_)).getOrElse(sqls"")
- val orderByClause = reversed.map(x =>
- if (x) sqls"eventTime desc" else sqls"eventTime asc"
- ).getOrElse(sqls"eventTime asc")
- val limitClause = limit.map(x =>
- if (x < 0) sqls"" else sqls.limit(x)
- ).getOrElse(sqls"")
- val q = sql"""
- select
- id,
- event,
- entityType,
- entityId,
- targetEntityType,
- targetEntityId,
- properties,
- eventTime,
- eventTimeZone,
- tags,
- prId,
- creationTime,
- creationTimeZone
- from $tableName
- $whereClause
- order by $orderByClause
- $limitClause
- """
- q.map(resultToEvent).list().apply().toIterator
- }
- }
-
- private[predictionio] def resultToEvent(rs: WrappedResultSet): Event = {
- Event(
- eventId = rs.stringOpt("id"),
- event = rs.string("event"),
- entityType = rs.string("entityType"),
- entityId = rs.string("entityId"),
- targetEntityType = rs.stringOpt("targetEntityType"),
- targetEntityId = rs.stringOpt("targetEntityId"),
- properties = rs.stringOpt("properties").map(p =>
- DataMap(read[JObject](p))).getOrElse(DataMap()),
- eventTime = new DateTime(rs.jodaDateTime("eventTime"),
- DateTimeZone.forID(rs.string("eventTimeZone"))),
- tags = rs.stringOpt("tags").map(t => t.split(",").toList).getOrElse(Nil),
- prId = rs.stringOpt("prId"),
- creationTime = new DateTime(rs.jodaDateTime("creationTime"),
- DateTimeZone.forID(rs.string("creationTimeZone")))
- )
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
deleted file mode 100644
index b48502a..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Model
-import org.apache.predictionio.data.storage.Models
-import org.apache.predictionio.data.storage.StorageClientConfig
-import scalikejdbc._
-
-/** JDBC implementation of [[Models]] */
-class JDBCModels(client: String, config: StorageClientConfig, prefix: String)
- extends Models with Logging {
- /** Database table name for this data access object */
- val tableName = JDBCUtils.prefixTableName(prefix, "models")
-
- /** Determines binary column type based on JDBC driver type */
- val binaryColumnType = JDBCUtils.binaryColumnType(client)
- DB autoCommit { implicit session =>
- sql"""
- create table if not exists $tableName (
- id varchar(100) not null primary key,
- models $binaryColumnType not null)""".execute().apply()
- }
-
- def insert(i: Model): Unit = DB localTx { implicit session =>
- sql"insert into $tableName values(${i.id}, ${i.models})".update().apply()
- }
-
- def get(id: String): Option[Model] = DB readOnly { implicit session =>
- sql"select id, models from $tableName where id = $id".map { r =>
- Model(id = r.string("id"), models = r.bytes("models"))
- }.single().apply()
- }
-
- def delete(id: String): Unit = DB localTx { implicit session =>
- sql"delete from $tableName where id = $id".execute().apply()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
deleted file mode 100644
index 2e6ee83..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import java.sql.{DriverManager, ResultSet}
-
-import com.github.nscala_time.time.Imports._
-import org.apache.predictionio.data.storage.{DataMap, Event, PEvents, StorageClientConfig}
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.{JdbcRDD, RDD}
-import org.apache.spark.sql.{SQLContext, SaveMode}
-import org.json4s.JObject
-import org.json4s.native.Serialization
-import scalikejdbc._
-
-/** JDBC implementation of [[PEvents]] */
-class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String) extends PEvents {
- @transient private implicit lazy val formats = org.json4s.DefaultFormats
- def find(
- appId: Int,
- channelId: Option[Int] = None,
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None,
- entityType: Option[String] = None,
- entityId: Option[String] = None,
- eventNames: Option[Seq[String]] = None,
- targetEntityType: Option[Option[String]] = None,
- targetEntityId: Option[Option[String]] = None)(sc: SparkContext): RDD[Event] = {
- val lower = startTime.map(_.getMillis).getOrElse(0.toLong)
- /** Change the default upper bound from +100 to +1 year because MySQL's
- * FROM_UNIXTIME(t) will return NULL if we use +100 years.
- */
- val upper = untilTime.map(_.getMillis).getOrElse((DateTime.now + 1.years).getMillis)
- val par = scala.math.min(
- new Duration(upper - lower).getStandardDays,
- config.properties.getOrElse("PARTITIONS", "4").toLong).toInt
- val entityTypeClause = entityType.map(x => s"and entityType = '$x'").getOrElse("")
- val entityIdClause = entityId.map(x => s"and entityId = '$x'").getOrElse("")
- val eventNamesClause =
- eventNames.map("and (" + _.map(y => s"event = '$y'").mkString(" or ") + ")").getOrElse("")
- val targetEntityTypeClause = targetEntityType.map(
- _.map(x => s"and targetEntityType = '$x'"
- ).getOrElse("and targetEntityType is null")).getOrElse("")
- val targetEntityIdClause = targetEntityId.map(
- _.map(x => s"and targetEntityId = '$x'"
- ).getOrElse("and targetEntityId is null")).getOrElse("")
- val q = s"""
- select
- id,
- event,
- entityType,
- entityId,
- targetEntityType,
- targetEntityId,
- properties,
- eventTime,
- eventTimeZone,
- tags,
- prId,
- creationTime,
- creationTimeZone
- from ${JDBCUtils.eventTableName(namespace, appId, channelId)}
- where
- eventTime >= ${JDBCUtils.timestampFunction(client)}(?) and
- eventTime < ${JDBCUtils.timestampFunction(client)}(?)
- $entityTypeClause
- $entityIdClause
- $eventNamesClause
- $targetEntityTypeClause
- $targetEntityIdClause
- """.replace("\n", " ")
- new JdbcRDD(
- sc,
- () => {
- DriverManager.getConnection(
- client,
- config.properties("USERNAME"),
- config.properties("PASSWORD"))
- },
- q,
- lower / 1000,
- upper / 1000,
- par,
- (r: ResultSet) => {
- Event(
- eventId = Option(r.getString("id")),
- event = r.getString("event"),
- entityType = r.getString("entityType"),
- entityId = r.getString("entityId"),
- targetEntityType = Option(r.getString("targetEntityType")),
- targetEntityId = Option(r.getString("targetEntityId")),
- properties = Option(r.getString("properties")).map(x =>
- DataMap(Serialization.read[JObject](x))).getOrElse(DataMap()),
- eventTime = new DateTime(r.getTimestamp("eventTime").getTime,
- DateTimeZone.forID(r.getString("eventTimeZone"))),
- tags = Option(r.getString("tags")).map(x =>
- x.split(",").toList).getOrElse(Nil),
- prId = Option(r.getString("prId")),
- creationTime = new DateTime(r.getTimestamp("creationTime").getTime,
- DateTimeZone.forID(r.getString("creationTimeZone"))))
- }).cache()
- }
-
- def write(events: RDD[Event], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
- val sqlContext = new SQLContext(sc)
-
- import sqlContext.implicits._
-
- val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
-
- val eventTableColumns = Seq[String](
- "id"
- , "event"
- , "entityType"
- , "entityId"
- , "targetEntityType"
- , "targetEntityId"
- , "properties"
- , "eventTime"
- , "eventTimeZone"
- , "tags"
- , "prId"
- , "creationTime"
- , "creationTimeZone")
-
- val eventDF = events.map(x =>
- Event(eventId = None, event = x.event, entityType = x.entityType,
- entityId = x.entityId, targetEntityType = x.targetEntityType,
- targetEntityId = x.targetEntityId, properties = x.properties,
- eventTime = x.eventTime, tags = x.tags, prId= x.prId,
- creationTime = x.eventTime)
- )
- .map { event =>
- (event.eventId.getOrElse(JDBCUtils.generateId)
- , event.event
- , event.entityType
- , event.entityId
- , event.targetEntityType.orNull
- , event.targetEntityId.orNull
- , if (!event.properties.isEmpty) Serialization.write(event.properties.toJObject) else null
- , new java.sql.Timestamp(event.eventTime.getMillis)
- , event.eventTime.getZone.getID
- , if (event.tags.nonEmpty) Some(event.tags.mkString(",")) else null
- , event.prId
- , new java.sql.Timestamp(event.creationTime.getMillis)
- , event.creationTime.getZone.getID)
- }.toDF(eventTableColumns:_*)
-
- // spark version 1.4.0 or higher
- val prop = new java.util.Properties
- prop.setProperty("user", config.properties("USERNAME"))
- prop.setProperty("password", config.properties("PASSWORD"))
- eventDF.write.mode(SaveMode.Append).jdbc(client, tableName, prop)
- }
-
- def delete(eventIds: RDD[String], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
-
- eventIds.foreachPartition{ iter =>
-
- iter.foreach { eventId =>
- DB localTx { implicit session =>
- val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
- val table = SQLSyntax.createUnsafely(tableName)
- sql"""
- delete from $table where id = $eventId
- """.update().apply()
- true
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
deleted file mode 100644
index 3eb55ba..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import scalikejdbc._
-
-/** JDBC related utilities */
-object JDBCUtils {
- /** Extract JDBC driver type from URL
- *
- * @param url JDBC URL
- * @return The driver type, e.g. postgresql
- */
- def driverType(url: String): String = {
- val capture = """jdbc:([^:]+):""".r
- capture findFirstIn url match {
- case Some(capture(driverType)) => driverType
- case None => ""
- }
- }
-
- /** Determines binary column type from JDBC URL
- *
- * @param url JDBC URL
- * @return Binary column type as SQLSyntax, e.g. LONGBLOB
- */
- def binaryColumnType(url: String): SQLSyntax = {
- driverType(url) match {
- case "postgresql" => sqls"bytea"
- case "mysql" => sqls"longblob"
- case _ => sqls"longblob"
- }
- }
-
- /** Determines UNIX timestamp conversion function from JDBC URL
- *
- * @param url JDBC URL
- * @return Timestamp conversion function, e.g. TO_TIMESTAMP
- */
- def timestampFunction(url: String): String = {
- driverType(url) match {
- case "postgresql" => "to_timestamp"
- case "mysql" => "from_unixtime"
- case _ => "from_unixtime"
- }
- }
-
- /** Converts Map of String to String to comma-separated list of key=value
- *
- * @param m Map of String to String
- * @return Comma-separated list, e.g. FOO=BAR,X=Y,...
- */
- def mapToString(m: Map[String, String]): String = {
- m.map(t => s"${t._1}=${t._2}").mkString(",")
- }
-
- /** Inverse of mapToString
- *
- * @param str Comma-separated list, e.g. FOO=BAR,X=Y,...
- * @return Map of String to String, e.g. Map("FOO" -> "BAR", "X" -> "Y", ...)
- */
- def stringToMap(str: String): Map[String, String] = {
- str.split(",").map { x =>
- val y = x.split("=")
- y(0) -> y(1)
- }.toMap[String, String]
- }
-
- /** Generate 32-character random ID using UUID with - stripped */
- def generateId: String = java.util.UUID.randomUUID().toString.replace("-", "")
-
- /** Prefix a table name
- *
- * @param prefix Table prefix
- * @param table Table name
- * @return Prefixed table name
- */
- def prefixTableName(prefix: String, table: String): SQLSyntax =
- sqls.createUnsafely(s"${prefix}_$table")
-
- /** Derive event table name
- *
- * @param namespace Namespace of event tables
- * @param appId App ID
- * @param channelId Optional channel ID
- * @return Full event table name
- */
- def eventTableName(namespace: String, appId: Int, channelId: Option[Int]): String =
- s"${namespace}_${appId}${channelId.map("_" + _).getOrElse("")}"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
deleted file mode 100644
index 661e05e..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.jdbc
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.BaseStorageClient
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.StorageClientException
-import scalikejdbc._
-
-/** JDBC implementation of [[BaseStorageClient]] */
-class StorageClient(val config: StorageClientConfig)
- extends BaseStorageClient with Logging {
- override val prefix = "JDBC"
-
- if (!config.properties.contains("URL")) {
- throw new StorageClientException("The URL variable is not set!", null)
- }
- if (!config.properties.contains("USERNAME")) {
- throw new StorageClientException("The USERNAME variable is not set!", null)
- }
- if (!config.properties.contains("PASSWORD")) {
- throw new StorageClientException("The PASSWORD variable is not set!", null)
- }
-
- // set max size of connection pool
- val maxSize: Int = config.properties.getOrElse("CONNECTIONS", "8").toInt
- val settings = ConnectionPoolSettings(maxSize = maxSize)
-
- ConnectionPool.singleton(
- config.properties("URL"),
- config.properties("USERNAME"),
- config.properties("PASSWORD"),
- settings)
- /** JDBC connection URL. Connections are managed by ScalikeJDBC. */
- val client = config.properties("URL")
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala b/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
deleted file mode 100644
index e552e54..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-/** JDBC implementation of storage traits, supporting meta data, event data, and
- * model data
- *
- * @group Implementation
- */
-package object jdbc {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala b/data/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
deleted file mode 100644
index f528af9..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.localfs
-
-import java.io.File
-import java.io.FileNotFoundException
-import java.io.FileOutputStream
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Model
-import org.apache.predictionio.data.storage.Models
-import org.apache.predictionio.data.storage.StorageClientConfig
-
-import scala.io.Source
-
-class LocalFSModels(f: File, config: StorageClientConfig, prefix: String)
- extends Models with Logging {
-
- def insert(i: Model): Unit = {
- try {
- val fos = new FileOutputStream(new File(f, s"${prefix}${i.id}"))
- fos.write(i.models)
- fos.close
- } catch {
- case e: FileNotFoundException => error(e.getMessage)
- }
- }
-
- def get(id: String): Option[Model] = {
- try {
- Some(Model(
- id = id,
- models = Source.fromFile(new File(f, s"${prefix}${id}"))(
- scala.io.Codec.ISO8859).map(_.toByte).toArray))
- } catch {
- case e: Throwable =>
- error(e.getMessage)
- None
- }
- }
-
- def delete(id: String): Unit = {
- val m = new File(f, s"${prefix}${id}")
- if (!m.delete) error(s"Unable to delete ${m.getCanonicalPath}!")
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala b/data/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
deleted file mode 100644
index b9ec957..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/StorageClient.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.localfs
-
-import java.io.File
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.BaseStorageClient
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.StorageClientException
-
-class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
- with Logging {
- override val prefix = "LocalFS"
- val f = new File(
- config.properties.getOrElse("PATH", config.properties("HOSTS")))
- if (f.exists) {
- if (!f.isDirectory) throw new StorageClientException(
- s"${f} already exists but it is not a directory!",
- null)
- if (!f.canWrite) throw new StorageClientException(
- s"${f} already exists but it is not writable!",
- null)
- } else {
- if (!f.mkdirs) throw new StorageClientException(
- s"${f} does not exist and automatic creation failed!",
- null)
- }
- val client = f
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala b/data/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
deleted file mode 100644
index 554ab26..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/localfs/package.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-/** Local file system implementation of storage traits, supporting model data only
- *
- * @group Implementation
- */
-package object localfs {}
[49/50] [abbrv] incubator-predictionio git commit: Add missing step
Posted by do...@apache.org.
Add missing step
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/991b88fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/991b88fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/991b88fe
Branch: refs/heads/livedoc
Commit: 991b88fec02791e70a0c6105d81656ae5121ceee
Parents: 5a35586
Author: Donald Szeto <do...@apache.org>
Authored: Sun Apr 9 11:37:09 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Apr 9 11:37:09 2017 -0700
----------------------------------------------------------------------
PMC.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/991b88fe/PMC.md
----------------------------------------------------------------------
diff --git a/PMC.md b/PMC.md
index 3609f34..a8be0b6 100644
--- a/PMC.md
+++ b/PMC.md
@@ -10,6 +10,7 @@ signing key.
1. `git checkout -b release/0.11.0`
2. Replace all `0.11.0-SNAPSHOT` in the code tree to `0.11.0-incubating`.
3. `git commit -am "Prepare 0.11.0-incubating-rc1"`
+ 4. `git tag -am "Apache PredictionIO (incubating) 0.11.0-rc1" v0.11.0-incubating-rc1`
4. If you have not done so, use SVN to checkout
https://dist.apache.org/repos/dist/dev/incubator/predictionio. This is the area
for staging release candidates for voting.
[23/50] [abbrv] incubator-predictionio git commit: [PIO-51] Properly
locate engine directory
Posted by do...@apache.org.
[PIO-51] Properly locate engine directory
When --engine-dir is passed as CLI argument, use this instead of user.dir
Closes #357
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/6d92b457
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/6d92b457
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/6d92b457
Branch: refs/heads/livedoc
Commit: 6d92b45763e9d8f1164ef0610562aa1c87cc96d9
Parents: 0d86465
Author: Chan Lee <ch...@gmail.com>
Authored: Mon Mar 13 09:57:35 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 13 09:57:35 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/tools/RunServer.scala | 5 +++--
.../scala/org/apache/predictionio/tools/RunWorkflow.scala | 10 ++++++----
.../org/apache/predictionio/tools/commands/Engine.scala | 3 ++-
.../org/apache/predictionio/tools/console/Console.scala | 8 +++-----
4 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6d92b457/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
index 1c70974..db055b5 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunServer.scala
@@ -47,7 +47,7 @@ case class ServerArgs(
eventServer: EventServerArgs = EventServerArgs(),
batch: String = "",
accessKey: String = "",
- variantJson: File = new File("engine.json"),
+ variantJson: Option[File] = None,
jsonExtractor: JsonExtractorOption = JsonExtractorOption.Both)
@@ -68,7 +68,8 @@ object RunServer extends Logging {
"--engineInstanceId",
engineInstanceId,
"--engine-variant",
- engineDirPath + File.separator + serverArgs.variantJson.getName,
+ serverArgs.variantJson.getOrElse(
+ new File(engineDirPath, "engine.json")).getCanonicalPath,
"--ip",
serverArgs.deploy.ip,
"--port",
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6d92b457/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
index 97831a6..462770e 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/RunWorkflow.scala
@@ -36,7 +36,7 @@ import scala.sys.process._
case class WorkflowArgs(
batch: String = "",
- variantJson: File = new File("engine.json"),
+ variantJson: Option[File] = None,
verbosity: Int = 0,
engineParamsKey: Option[String] = None,
engineFactory: Option[String] = None,
@@ -56,15 +56,17 @@ object RunWorkflow extends Logging {
verbose: Boolean = false): Expected[(Process, () => Unit)] = {
val jarFiles = jarFilesForScala(engineDirPath).map(_.toURI)
- val variantJson = engineDirPath + File.separator + wa.variantJson.getName
- val ei = Console.getEngineInfo(new File(variantJson))
+ val variantJson = wa.variantJson.getOrElse(new File(engineDirPath, "engine.json"))
+ val ei = Console.getEngineInfo(
+ variantJson,
+ engineDirPath)
val args = Seq(
"--engine-id",
ei.engineId,
"--engine-version",
ei.engineVersion,
"--engine-variant",
- variantJson,
+ variantJson.toURI.toString,
"--verbosity",
wa.verbosity.toString) ++
wa.engineFactory.map(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6d92b457/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
index 69a3924..c4dc8d4 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Engine.scala
@@ -222,7 +222,8 @@ object Engine extends EitherLogging {
return Left(verifyResult.left.get)
}
val ei = Console.getEngineInfo(
- new File(engineDirPath, serverArgs.variantJson.getName))
+ serverArgs.variantJson.getOrElse(new File(engineDirPath, "engine.json")),
+ engineDirPath)
val engineInstances = storage.Storage.getMetaDataEngineInstances
val engineInstance = engineInstanceId map { eid =>
engineInstances.get(eid)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/6d92b457/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
index f4c891a..810fee0 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/console/Console.scala
@@ -120,10 +120,9 @@ object Console extends Logging {
"deployment.")
opt[String]("engine-dir") abbr("ed") action { (x, c) =>
c.copy(engine = c.engine.copy(engineDir = Some(x)))
- } text("Specify absolute path for engine directory, default to " +
- "current directory.")
+ } text("Specify path for engine directory, default to current directory.")
opt[File]("variant") abbr("v") action { (x, c) =>
- c.copy(workflow = c.workflow.copy(variantJson = x))
+ c.copy(workflow = c.workflow.copy(variantJson = Some(x)))
}
opt[File]("sbt") action { (x, c) =>
c.copy(build = c.build.copy(sbt = Some(x)))
@@ -734,7 +733,7 @@ object Console extends Logging {
}
}
- def getEngineInfo(jsonFile: File): EngineInfo = {
+ def getEngineInfo(jsonFile: File, engineDir: String): EngineInfo = {
// Use engineFactory as engineId
val variantJson = parse(Source.fromFile(jsonFile).mkString)
val engineId = variantJson \ "engineFactory" match {
@@ -754,7 +753,6 @@ object Console extends Logging {
}
// Use hash of engine directory as engineVersion
- val engineDir = sys.props("user.dir")
val engineVersion = java.security.MessageDigest.getInstance("SHA-1").
digest(engineDir.getBytes).map("%02x".format(_)).mkString
[41/50] [abbrv] incubator-predictionio git commit: Update release
notes and publish storage JARs
Posted by do...@apache.org.
Update release notes and publish storage JARs
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/f8795ca4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/f8795ca4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/f8795ca4
Branch: refs/heads/livedoc
Commit: f8795ca499a2331302a0904e01c87d9e00944798
Parents: d0d2570
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 30 15:24:34 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 30 15:24:34 2017 -0700
----------------------------------------------------------------------
RELEASE.md | 5 ++++-
build.sbt | 19 +++++++------------
make-distribution.sh | 2 +-
3 files changed, 12 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f8795ca4/RELEASE.md
----------------------------------------------------------------------
diff --git a/RELEASE.md b/RELEASE.md
index 3cc9259..db3549b 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -44,7 +44,10 @@ Apr ?, 2017
parameter if you want to run `pio build`, `pio train`, or `pio deploy` outside
of an engine directory. This is an interim solution before an engine registry
feature becomes available in the future.
-- PIO-54: Generated access keys will no longer start with a `-` character.
+- PIO-49: PostgreSQL JDBC driver is no longer bundled with the core assembly. If
+ you are using PostgreSQL, you must download the JDBC driver and update your
+ configuration to point to the correct JDBC driver file.
+- PIO-54: New generated access keys will no longer start with a `-` character.
#### Other Changes
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f8795ca4/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 06030cb..a0a41dd 100644
--- a/build.sbt
+++ b/build.sbt
@@ -108,32 +108,26 @@ val commonTestSettings = Seq(
val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
settings(commonSettings: _*).
- enablePlugins(GenJavadocPlugin).
- settings(publishArtifact := false)
+ enablePlugins(GenJavadocPlugin)
val dataElasticsearch = (project in file("storage/elasticsearch")).
- settings(commonSettings: _*).
- settings(publishArtifact := false)
+ settings(commonSettings: _*)
val dataHbase = (project in file("storage/hbase")).
settings(commonSettings: _*).
- enablePlugins(GenJavadocPlugin).
- settings(publishArtifact := false)
+ enablePlugins(GenJavadocPlugin)
val dataHdfs = (project in file("storage/hdfs")).
settings(commonSettings: _*).
- enablePlugins(GenJavadocPlugin).
- settings(publishArtifact := false)
+ enablePlugins(GenJavadocPlugin)
val dataJdbc = (project in file("storage/jdbc")).
settings(commonSettings: _*).
- enablePlugins(GenJavadocPlugin).
- settings(publishArtifact := false)
+ enablePlugins(GenJavadocPlugin)
val dataLocalfs = (project in file("storage/localfs")).
settings(commonSettings: _*).
- enablePlugins(GenJavadocPlugin).
- settings(publishArtifact := false)
+ enablePlugins(GenJavadocPlugin)
val common = (project in file("common")).
settings(commonSettings: _*).
@@ -184,6 +178,7 @@ val storageSubprojects = Seq(
val storage = (project in file("storage"))
.aggregate(storageSubprojects map Project.projectToRef: _*)
.disablePlugins(sbtassembly.AssemblyPlugin)
+ .settings(publishArtifact := false)
val assembly = (project in file("assembly")).
settings(commonSettings: _*)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/f8795ca4/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 03c15e8..09025ee 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -69,7 +69,7 @@ cd ${FWDIR}
set -x
sbt/sbt "${JAVA_PROPS[@]}" clean
sbt/sbt "${JAVA_PROPS[@]}" printBuildInfo
-sbt/sbt "${JAVA_PROPS[@]}" publishLocal assembly storage/assembly
+sbt/sbt "${JAVA_PROPS[@]}" publishLocal assembly storage/publishLocal storage/assembly
sbt/sbt "${JAVA_PROPS[@]}" assembly/clean assembly/universal:packageBin assembly/universal:packageZipTarball
if [ x$RPM_BUILD = "xtrue" ] ; then
sbt/sbt "${JAVA_PROPS[@]}" assembly/rpm:packageBin
[08/50] [abbrv] incubator-predictionio git commit: [PIO-48]
scala-local-helloword Sample Not working
Posted by do...@apache.org.
[PIO-48] scala-local-helloword Sample Not working
Closes #332
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/19c47adf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/19c47adf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/19c47adf
Branch: refs/heads/livedoc
Commit: 19c47adf553ad6c70c4020db9e50d65ec561e642
Parents: aebe43a
Author: Rajdeep Dua <du...@yahoo.com>
Authored: Tue Mar 7 10:50:45 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 10:50:45 2017 -0800
----------------------------------------------------------------------
.../scala-local-helloworld/HelloWorld.scala | 20 +++++++++++++++-----
.../scala-local-helloworld/build.sbt | 5 +++--
2 files changed, 18 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/19c47adf/examples/experimental/scala-local-helloworld/HelloWorld.scala
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/HelloWorld.scala b/examples/experimental/scala-local-helloworld/HelloWorld.scala
index bc565ce..6ec4f0a 100644
--- a/examples/experimental/scala-local-helloworld/HelloWorld.scala
+++ b/examples/experimental/scala-local-helloworld/HelloWorld.scala
@@ -42,11 +42,17 @@ class MyPredictedResult(
val temperature: Double
) extends Serializable
-// controller components
-class MyDataSource extends LDataSource[EmptyDataSourceParams, EmptyDataParams,
- MyTrainingData, MyQuery, EmptyActualResult] {
+case class MyDataSourceParams(val multiplier: Int
+ ) extends Params
+
+class MyDataSource extends LDataSource[
+ MyTrainingData,
+ EmptyEvaluationInfo,
+ MyQuery,
+ EmptyActualResult] {
/* override this to return Training Data only */
+
override
def readTraining(): MyTrainingData = {
val lines = Source.fromFile("../data/helloworld/data.csv").getLines()
@@ -59,8 +65,12 @@ class MyDataSource extends LDataSource[EmptyDataSourceParams, EmptyDataParams,
}
}
-class MyAlgorithm extends LAlgorithm[EmptyAlgorithmParams, MyTrainingData,
- MyModel, MyQuery, MyPredictedResult] {
+class MyAlgorithm extends LAlgorithm[
+ MyTrainingData,
+ MyModel,
+ MyQuery,
+ MyPredictedResult] {
+
override
def train(pd: MyTrainingData): MyModel = {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/19c47adf/examples/experimental/scala-local-helloworld/build.sbt
----------------------------------------------------------------------
diff --git a/examples/experimental/scala-local-helloworld/build.sbt b/examples/experimental/scala-local-helloworld/build.sbt
index df7ba52..e14c05b 100644
--- a/examples/experimental/scala-local-helloworld/build.sbt
+++ b/examples/experimental/scala-local-helloworld/build.sbt
@@ -24,6 +24,7 @@ name := "example-scala-local-helloworld"
organization := "org.sample"
libraryDependencies ++= Seq(
- "org.apache.predictionio" %% "core" % "0.9.1" % "provided",
- "org.apache.predictionio" %% "data" % "0.9.1" % "provided",
+ "org.slf4j" % "slf4j-api" % "1.6.1",
+ "org.apache.predictionio" %% "apache-predictionio-data" % "0.10.0-incubating",
+ "org.apache.predictionio" %% "apache-predictionio-core" % "0.10.0-incubating",
"org.apache.spark" %% "spark-core" % "1.2.0" % "provided")
[34/50] [abbrv] incubator-predictionio git commit: Fix tests for
Scala 2.11
Posted by do...@apache.org.
Fix tests for Scala 2.11
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/14b49982
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/14b49982
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/14b49982
Branch: refs/heads/livedoc
Commit: 14b49982650965c626788206aedea511f033033d
Parents: 68708b4
Author: Donald Szeto <do...@apache.org>
Authored: Mon Mar 20 21:41:52 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 20 21:41:52 2017 -0700
----------------------------------------------------------------------
common/build.sbt | 3 ++-
tests/Dockerfile | 2 ++
tests/build_docker.sh | 1 +
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/14b49982/common/build.sbt
----------------------------------------------------------------------
diff --git a/common/build.sbt b/common/build.sbt
index 32afb97..19e4f04 100644
--- a/common/build.sbt
+++ b/common/build.sbt
@@ -22,6 +22,7 @@ name := "apache-predictionio-common"
libraryDependencies ++= Seq(
"io.spray" %% "spray-can" % "1.3.3",
"io.spray" %% "spray-routing" % "1.3.3",
- "com.typesafe.akka" %% "akka-actor" % akkaVersion.value)
+ "com.typesafe.akka" %% "akka-actor" % akkaVersion.value,
+ "com.typesafe.akka" %% "akka-slf4j" % akkaVersion.value)
pomExtra := childrenPomExtra.value
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/14b49982/tests/Dockerfile
----------------------------------------------------------------------
diff --git a/tests/Dockerfile b/tests/Dockerfile
index 619bdf8..f7dbd68 100644
--- a/tests/Dockerfile
+++ b/tests/Dockerfile
@@ -22,10 +22,12 @@ ARG SPARK_DIR
ARG PGSQL_JAR
ARG PIO_SCALA_VERSION
ARG PIO_SPARK_VERSION
+ARG PIO_HADOOP_VERSION
ARG PIO_ELASTICSEARCH_VERSION
ENV PIO_SCALA_VERSION=$PIO_SCALA_VERSION
ENV PIO_SPARK_VERSION=$PIO_SPARK_VERSION
+ENV PIO_HADOOP_VERSION=$PIO_HADOOP_VERSION
ENV PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION
ENV PGSQL_JAR=$PGSQL_JAR
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/14b49982/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index af30cb8..27fb744 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -54,4 +54,5 @@ docker build -t predictionio/pio-testing $DIR \
--build-arg PGSQL_JAR=$PGSQL_JAR \
--build-arg PIO_SCALA_VERSION=$PIO_SCALA_VERSION \
--build-arg PIO_SPARK_VERSION=$PIO_SPARK_VERSION \
+ --build-arg PIO_HADOOP_VERSION=$PIO_HADOOP_VERSION \
--build-arg PIO_ELASTICSEARCH_VERSION=$PIO_ELASTICSEARCH_VERSION
[17/50] [abbrv] incubator-predictionio git commit: Fix Travis
Posted by do...@apache.org.
Fix Travis
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/2d388db2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/2d388db2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/2d388db2
Branch: refs/heads/livedoc
Commit: 2d388db27b82166ebc3993adab452e4b2e38dd68
Parents: 31c4bd1
Author: Donald Szeto <do...@apache.org>
Authored: Wed Mar 8 20:29:06 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Wed Mar 8 20:29:06 2017 -0800
----------------------------------------------------------------------
tests/build-docker.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/2d388db2/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index a899a18..4438396 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash -ex
+#!/bin/bash -ex
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
[24/50] [abbrv] incubator-predictionio git commit: Integration tests
for both ES1 and ES5
Posted by do...@apache.org.
Integration tests for both ES1 and ES5
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/3da809c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/3da809c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/3da809c7
Branch: refs/heads/livedoc
Commit: 3da809c7f6b82add7007d19707bfd312ec68bb1a
Parents: 6d92b45
Author: Donald Szeto <do...@apache.org>
Authored: Mon Mar 13 14:52:24 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 13 14:52:24 2017 -0700
----------------------------------------------------------------------
tests/Dockerfile-es1 | 57 +++++++++++++
tests/after_script.travis.sh | 8 +-
tests/before_script.travis.sh | 4 +-
tests/build-docker.sh | 8 +-
tests/docker-compose-es1.yml | 38 +++++++++
tests/docker-files/env-conf/pio-env-es1.sh | 103 ++++++++++++++++++++++++
tests/run_docker.sh | 19 +++--
7 files changed, 229 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/Dockerfile-es1
----------------------------------------------------------------------
diff --git a/tests/Dockerfile-es1 b/tests/Dockerfile-es1
new file mode 100644
index 0000000..73d4c36
--- /dev/null
+++ b/tests/Dockerfile-es1
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM predictionio/pio
+
+ENV SPARK_VERSION 1.6.3
+ENV ELASTICSEARCH_VERSION 1.7
+ENV HBASE_VERSION 1.0.0
+
+ADD docker-files/spark-${SPARK_VERSION}-bin-hadoop2.6.tgz /vendors
+# WORKAROUND: es-hadoop stops on RDD#take(1)
+RUN echo "spark.locality.wait.node 0s" > /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6/conf/spark-defaults.conf
+ENV SPARK_HOME /vendors/spark-${SPARK_VERSION}-bin-hadoop2.6
+
+COPY docker-files/postgresql-9.4-1204.jdbc41.jar /drivers/postgresql-9.4-1204.jdbc41.jar
+COPY docker-files/init.sh init.sh
+COPY docker-files/env-conf/hbase-site.xml ${PIO_HOME}/conf/hbase-site.xml
+COPY docker-files/env-conf/pio-env-es1.sh ${PIO_HOME}/conf/pio-env.sh
+COPY docker-files/pgpass /root/.pgpass
+RUN chmod 600 /root/.pgpass
+
+# Python
+RUN pip install python-dateutil
+RUN pip install pytz
+
+# Default repositories setup
+ENV PIO_STORAGE_REPOSITORIES_METADATA_SOURCE PGSQL
+ENV PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE PGSQL
+ENV PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE PGSQL
+
+# JVM settings
+ENV JVM_OPTS '-Dfile.encoding=UTF8 -Xms2048M -Xmx2048M -Xss8M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=256M'
+
+# Expose relevant ports
+# pio engine
+EXPOSE 8000
+# eventserver
+EXPOSE 7070
+
+ENV SLEEP_TIME 30
+
+ENTRYPOINT ["/init.sh"]
+CMD 'bash'
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index 005d410..8cbe2c0 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -16,8 +16,14 @@
# limitations under the License.
#
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
# Print a summary of containers used
docker ps -a
# Clean up used containers
-docker-compose -f tests/docker-compose.yml down
+if [ "$ES_VERSION" = "1" ]; then
+ docker-compose -f $DIR/docker-compose-es1.yml down
+else
+ docker-compose -f $DIR/docker-compose.yml down
+fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/before_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/before_script.travis.sh b/tests/before_script.travis.sh
index b6d3966..9ec7a4a 100755
--- a/tests/before_script.travis.sh
+++ b/tests/before_script.travis.sh
@@ -16,4 +16,6 @@
# limitations under the License.
#
-tests/build-docker.sh
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+$DIR/build-docker.sh
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/build-docker.sh
----------------------------------------------------------------------
diff --git a/tests/build-docker.sh b/tests/build-docker.sh
index 4438396..ed43715 100755
--- a/tests/build-docker.sh
+++ b/tests/build-docker.sh
@@ -39,6 +39,12 @@ mkdir assembly
cp dist/lib/*.jar assembly/
mkdir -p lib/spark
cp dist/lib/spark/*.jar lib/spark
+rm *.tar.gz
docker build -t predictionio/pio .
popd
-docker build -t predictionio/pio-testing $DIR
+
+if [ "$ES_VERSION" = "1" ]; then
+ docker build -t predictionio/pio-testing-es1 -f $DIR/Dockerfile-es1 $DIR
+else
+ docker build -t predictionio/pio-testing $DIR
+fi
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-compose-es1.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose-es1.yml b/tests/docker-compose-es1.yml
new file mode 100644
index 0000000..381f41c
--- /dev/null
+++ b/tests/docker-compose-es1.yml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2"
+services:
+ elasticsearch:
+ image: elasticsearch:1
+ environment:
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ hbase:
+ image: harisekhon/hbase:1.0
+ postgres:
+ image: postgres:9
+ environment:
+ POSTGRES_USER: pio
+ POSTGRES_PASSWORD: pio
+ POSTGRES_INITDB_ARGS: --encoding=UTF8
+ pio-testing:
+ image: predictionio/pio-testing-es1:latest
+ depends_on:
+ - elasticsearch
+ - hbase
+ - postgres
+ volumes:
+ - ~/.ivy2:/root/.ivy2
+ - ~/.sbt:/root/.sbt
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/docker-files/env-conf/pio-env-es1.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env-es1.sh b/tests/docker-files/env-conf/pio-env-es1.sh
new file mode 100644
index 0000000..e1076ba
--- /dev/null
+++ b/tests/docker-files/env-conf/pio-env-es1.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+#
+# Copy this file as pio-env.sh and edit it for your site's configuration.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# PredictionIO Main Configuration
+#
+# This section controls core behavior of PredictionIO. It is very likely that
+# you need to change these to fit your site.
+
+# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
+SPARK_HOME=$SPARK_HOME
+
+POSTGRES_JDBC_DRIVER=/drivers/postgresql-9.4-1204.jdbc41.jar
+MYSQL_JDBC_DRIVER=
+
+# ES_CONF_DIR: You must configure this if you have advanced configuration for
+# your Elasticsearch setup.
+# ES_CONF_DIR=/opt/elasticsearch
+
+# HADOOP_CONF_DIR: You must configure this if you intend to run PredictionIO
+# with Hadoop 2.
+# HADOOP_CONF_DIR=/opt/hadoop
+
+# HBASE_CONF_DIR: You must configure this if you intend to run PredictionIO
+# with HBase on a remote cluster.
+HBASE_CONF_DIR=$PIO_HOME/conf
+
+# Filesystem paths where PredictionIO uses as block storage.
+PIO_FS_BASEDIR=$HOME/.pio_store
+PIO_FS_ENGINESDIR=$PIO_FS_BASEDIR/engines
+PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp
+
+# PredictionIO Storage Configuration
+#
+# This section controls programs that make use of PredictionIO's built-in
+# storage facilities. Default values are shown below.
+#
+# For more information on storage configuration please refer to
+# https://docs.prediction.io/system/anotherdatastore/
+
+# Storage Repositories
+
+# Default is to use PostgreSQL
+PIO_STORAGE_REPOSITORIES_METADATA_NAME=pio_meta
+PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$PIO_STORAGE_REPOSITORIES_METADATA_SOURCE
+
+PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME=pio_event
+PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE
+
+PIO_STORAGE_REPOSITORIES_MODELDATA_NAME=pio_model
+PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE
+
+# Storage Data Sources
+
+# PostgreSQL Default Settings
+# Please change "pio" to your database name in PIO_STORAGE_SOURCES_PGSQL_URL
+# Please change PIO_STORAGE_SOURCES_PGSQL_USERNAME and
+# PIO_STORAGE_SOURCES_PGSQL_PASSWORD accordingly
+PIO_STORAGE_SOURCES_PGSQL_TYPE=jdbc
+PIO_STORAGE_SOURCES_PGSQL_URL=jdbc:postgresql://postgres/pio
+PIO_STORAGE_SOURCES_PGSQL_USERNAME=pio
+PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
+
+# MySQL Example
+# PIO_STORAGE_SOURCES_MYSQL_TYPE=jdbc
+# PIO_STORAGE_SOURCES_MYSQL_URL=jdbc:mysql://localhost/pio
+# PIO_STORAGE_SOURCES_MYSQL_USERNAME=pio
+# PIO_STORAGE_SOURCES_MYSQL_PASSWORD=pio
+
+# Elasticsearch Example
+PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
+#PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=pio
+PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=elasticsearch
+PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
+#PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$ELASTICSEARCH_HOME
+
+# Local File System Example
+PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
+PIO_STORAGE_SOURCES_LOCALFS_PATH=$PIO_FS_BASEDIR/local_models
+
+# HBase Example
+PIO_STORAGE_SOURCES_HBASE_TYPE=hbase
+#PIO_STORAGE_SOURCES_HBASE_HOME=$HBASE_HOME
+
+# HDFS config
+PIO_STORAGE_SOURCES_HDFS_TYPE=hdfs
+PIO_STORAGE_SOURCES_HDFS_PATH=/hdfs_models
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/3da809c7/tests/run_docker.sh
----------------------------------------------------------------------
diff --git a/tests/run_docker.sh b/tests/run_docker.sh
index fe07957..6257fa2 100755
--- a/tests/run_docker.sh
+++ b/tests/run_docker.sh
@@ -46,8 +46,17 @@ shift
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-docker-compose -f $DIR/docker-compose.yml run \
- -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
- -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
- -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
- pio-testing $@
+if [ "$ES_VERSION" = "1" ]; then
+ docker-compose -f $DIR/docker-compose-es1.yml run \
+ -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+ -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+ -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+ pio-testing $@
+else
+ docker-compose -f $DIR/docker-compose.yml run \
+ -e PIO_STORAGE_REPOSITORIES_METADATA_SOURCE=$META \
+ -e PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE=$EVENT \
+ -e PIO_STORAGE_REPOSITORIES_MODELDATA_SOURCE=$MODEL \
+ pio-testing $@
+fi
+
[12/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
new file mode 100644
index 0000000..077168a
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.AccessKey
+import org.apache.predictionio.data.storage.AccessKeys
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.elasticsearch.index.query.FilterBuilders._
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+
+import scala.util.Random
+
+/** Elasticsearch implementation of AccessKeys. */
+class ESAccessKeys(client: Client, config: StorageClientConfig, index: String)
+ extends AccessKeys with Logging {
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "accesskeys"
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val json =
+ (estype ->
+ ("properties" ->
+ ("key" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("events" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(json))).get
+ }
+
+ def insert(accessKey: AccessKey): Option[String] = {
+ val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
+ update(accessKey.copy(key = key))
+ Some(key)
+ }
+
+ def get(key: String): Option[AccessKey] = {
+ try {
+ val response = client.prepareGet(
+ index,
+ estype,
+ key).get()
+ Some(read[AccessKey](response.getSourceAsString))
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ case e: NullPointerException => None
+ }
+ }
+
+ def getAll(): Seq[AccessKey] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype)
+ ESUtils.getAll[AccessKey](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq[AccessKey]()
+ }
+ }
+
+ def getByAppid(appid: Int): Seq[AccessKey] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype).
+ setPostFilter(termFilter("appid", appid))
+ ESUtils.getAll[AccessKey](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq[AccessKey]()
+ }
+ }
+
+ def update(accessKey: AccessKey): Unit = {
+ try {
+ client.prepareIndex(index, estype, accessKey.key).setSource(write(accessKey)).get()
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ }
+ }
+
+ def delete(key: String): Unit = {
+ try {
+ client.prepareDelete(index, estype, key).get
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
new file mode 100644
index 0000000..3781a4b
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.App
+import org.apache.predictionio.data.storage.Apps
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.elasticsearch.index.query.FilterBuilders._
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+
+/** Elasticsearch implementation of Items. */
+class ESApps(client: Client, config: StorageClientConfig, index: String)
+ extends Apps with Logging {
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "apps"
+ private val seq = new ESSequences(client, config, index)
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val json =
+ (estype ->
+ ("properties" ->
+ ("name" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(json))).get
+ }
+
+ def insert(app: App): Option[Int] = {
+ val id =
+ if (app.id == 0) {
+ var roll = seq.genNext("apps")
+ while (!get(roll).isEmpty) roll = seq.genNext("apps")
+ roll
+ }
+ else app.id
+ val realapp = app.copy(id = id)
+ update(realapp)
+ Some(id)
+ }
+
+ def get(id: Int): Option[App] = {
+ try {
+ val response = client.prepareGet(
+ index,
+ estype,
+ id.toString).get()
+ Some(read[App](response.getSourceAsString))
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ case e: NullPointerException => None
+ }
+ }
+
+ def getByName(name: String): Option[App] = {
+ try {
+ val response = client.prepareSearch(index).setTypes(estype).
+ setPostFilter(termFilter("name", name)).get
+ val hits = response.getHits().hits()
+ if (hits.size > 0) {
+ Some(read[App](hits.head.getSourceAsString))
+ } else {
+ None
+ }
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ }
+ }
+
+ def getAll(): Seq[App] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype)
+ ESUtils.getAll[App](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq[App]()
+ }
+ }
+
+ def update(app: App): Unit = {
+ try {
+ val response = client.prepareIndex(index, estype, app.id.toString).
+ setSource(write(app)).get()
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ }
+ }
+
+ def delete(id: Int): Unit = {
+ try {
+ client.prepareDelete(index, estype, id.toString).get
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
new file mode 100644
index 0000000..52697fd
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Channel
+import org.apache.predictionio.data.storage.Channels
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.elasticsearch.index.query.FilterBuilders.termFilter
+import org.json4s.DefaultFormats
+import org.json4s.JsonDSL._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+
+class ESChannels(client: Client, config: StorageClientConfig, index: String)
+ extends Channels with Logging {
+
+ implicit val formats = DefaultFormats.lossless
+ private val estype = "channels"
+ private val seq = new ESSequences(client, config, index)
+ private val seqName = "channels"
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val json =
+ (estype ->
+ ("properties" ->
+ ("name" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(json))).get
+ }
+
+ def insert(channel: Channel): Option[Int] = {
+ val id =
+ if (channel.id == 0) {
+ var roll = seq.genNext(seqName)
+ while (!get(roll).isEmpty) roll = seq.genNext(seqName)
+ roll
+ } else channel.id
+
+ val realChannel = channel.copy(id = id)
+ if (update(realChannel)) Some(id) else None
+ }
+
+ def get(id: Int): Option[Channel] = {
+ try {
+ val response = client.prepareGet(
+ index,
+ estype,
+ id.toString).get()
+ Some(read[Channel](response.getSourceAsString))
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ case e: NullPointerException => None
+ }
+ }
+
+ def getByAppid(appid: Int): Seq[Channel] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype).
+ setPostFilter(termFilter("appid", appid))
+ ESUtils.getAll[Channel](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq[Channel]()
+ }
+ }
+
+ def update(channel: Channel): Boolean = {
+ try {
+ val response = client.prepareIndex(index, estype, channel.id.toString).
+ setSource(write(channel)).get()
+ true
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ false
+ }
+ }
+
+ def delete(id: Int): Unit = {
+ try {
+ client.prepareDelete(index, estype, id.toString).get
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
new file mode 100644
index 0000000..21690bf
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.EngineInstance
+import org.apache.predictionio.data.storage.EngineInstanceSerializer
+import org.apache.predictionio.data.storage.EngineInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.elasticsearch.index.query.FilterBuilders._
+import org.elasticsearch.search.sort.SortOrder
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+
+class ESEngineInstances(client: Client, config: StorageClientConfig, index: String)
+ extends EngineInstances with Logging {
+ implicit val formats = DefaultFormats + new EngineInstanceSerializer
+ private val estype = "engine_instances"
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val json =
+ (estype ->
+ ("properties" ->
+ ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("engineId" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("engineVersion" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("engineVariant" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("engineFactory" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("batch" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("dataSourceParams" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("preparatorParams" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("algorithmsParams" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("servingParams" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(json))).get
+ }
+
+ def insert(i: EngineInstance): String = {
+ try {
+ val response = client.prepareIndex(index, estype).
+ setSource(write(i)).get
+ response.getId
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ ""
+ }
+ }
+
+ def get(id: String): Option[EngineInstance] = {
+ try {
+ val response = client.prepareGet(index, estype, id).get
+ if (response.isExists) {
+ Some(read[EngineInstance](response.getSourceAsString))
+ } else {
+ None
+ }
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ }
+ }
+
+ def getAll(): Seq[EngineInstance] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype)
+ ESUtils.getAll[EngineInstance](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq()
+ }
+ }
+
+ def getCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Seq[EngineInstance] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype).setPostFilter(
+ andFilter(
+ termFilter("status", "COMPLETED"),
+ termFilter("engineId", engineId),
+ termFilter("engineVersion", engineVersion),
+ termFilter("engineVariant", engineVariant))).
+ addSort("startTime", SortOrder.DESC)
+ ESUtils.getAll[EngineInstance](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq()
+ }
+ }
+
+ def getLatestCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Option[EngineInstance] =
+ getCompleted(
+ engineId,
+ engineVersion,
+ engineVariant).headOption
+
+ def update(i: EngineInstance): Unit = {
+ try {
+ client.prepareUpdate(index, estype, i.id).setDoc(write(i)).get
+ } catch {
+ case e: ElasticsearchException => error(e.getMessage)
+ }
+ }
+
+ def delete(id: String): Unit = {
+ try {
+ val response = client.prepareDelete(index, estype, id).get
+ } catch {
+ case e: ElasticsearchException => error(e.getMessage)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
new file mode 100644
index 0000000..85bf820
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.EvaluationInstance
+import org.apache.predictionio.data.storage.EvaluationInstanceSerializer
+import org.apache.predictionio.data.storage.EvaluationInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.elasticsearch.index.query.FilterBuilders._
+import org.elasticsearch.search.sort.SortOrder
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+
+class ESEvaluationInstances(client: Client, config: StorageClientConfig, index: String)
+ extends EvaluationInstances with Logging {
+ implicit val formats = DefaultFormats + new EvaluationInstanceSerializer
+ private val estype = "evaluation_instances"
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val json =
+ (estype ->
+ ("properties" ->
+ ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("startTime" -> ("type" -> "date")) ~
+ ("endTime" -> ("type" -> "date")) ~
+ ("evaluationClass" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("engineParamsGeneratorClass" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("batch" ->
+ ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
+ ("evaluatorResults" ->
+ ("type" -> "string") ~ ("index" -> "no")) ~
+ ("evaluatorResultsHTML" ->
+ ("type" -> "string") ~ ("index" -> "no")) ~
+ ("evaluatorResultsJSON" ->
+ ("type" -> "string") ~ ("index" -> "no"))))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(json))).get
+ }
+
+ def insert(i: EvaluationInstance): String = {
+ try {
+ val response = client.prepareIndex(index, estype).
+ setSource(write(i)).get
+ response.getId
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ ""
+ }
+ }
+
+ def get(id: String): Option[EvaluationInstance] = {
+ try {
+ val response = client.prepareGet(index, estype, id).get
+ if (response.isExists) {
+ Some(read[EvaluationInstance](response.getSourceAsString))
+ } else {
+ None
+ }
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ None
+ }
+ }
+
+ def getAll(): Seq[EvaluationInstance] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype)
+ ESUtils.getAll[EvaluationInstance](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq()
+ }
+ }
+
+ def getCompleted(): Seq[EvaluationInstance] = {
+ try {
+ val builder = client.prepareSearch(index).setTypes(estype).setPostFilter(
+ termFilter("status", "EVALCOMPLETED")).
+ addSort("startTime", SortOrder.DESC)
+ ESUtils.getAll[EvaluationInstance](client, builder)
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ Seq()
+ }
+ }
+
+ def update(i: EvaluationInstance): Unit = {
+ try {
+ client.prepareUpdate(index, estype, i.id).setDoc(write(i)).get
+ } catch {
+ case e: ElasticsearchException => error(e.getMessage)
+ }
+ }
+
+ def delete(id: String): Unit = {
+ try {
+ client.prepareDelete(index, estype, id).get
+ } catch {
+ case e: ElasticsearchException => error(e.getMessage)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
new file mode 100644
index 0000000..5c9e170
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.elasticsearch.ElasticsearchException
+import org.elasticsearch.client.Client
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods._
+
+class ESSequences(client: Client, config: StorageClientConfig, index: String) extends Logging {
+ implicit val formats = DefaultFormats
+ private val estype = "sequences"
+
+ val indices = client.admin.indices
+ val indexExistResponse = indices.prepareExists(index).get
+ if (!indexExistResponse.isExists) {
+ // val settingsJson =
+ // ("number_of_shards" -> 1) ~
+ // ("auto_expand_replicas" -> "0-all")
+ indices.prepareCreate(index).get
+ }
+ val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
+ if (!typeExistResponse.isExists) {
+ val mappingJson =
+ (estype ->
+ ("_source" -> ("enabled" -> 0)) ~
+ ("_all" -> ("enabled" -> 0)) ~
+ ("_type" -> ("index" -> "no")) ~
+ ("enabled" -> 0))
+ indices.preparePutMapping(index).setType(estype).
+ setSource(compact(render(mappingJson))).get
+ }
+
+ def genNext(name: String): Int = {
+ try {
+ val response = client.prepareIndex(index, estype, name).
+ setSource(compact(render("n" -> name))).get
+ response.getVersion().toInt
+ } catch {
+ case e: ElasticsearchException =>
+ error(e.getMessage)
+ 0
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
new file mode 100644
index 0000000..f5c99bf
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import org.elasticsearch.action.search.SearchRequestBuilder
+import org.elasticsearch.client.Client
+import org.elasticsearch.common.unit.TimeValue
+import org.json4s.Formats
+import org.json4s.native.Serialization.read
+
+import scala.collection.mutable.ArrayBuffer
+
+object ESUtils {
+ val scrollLife = new TimeValue(60000)
+
+ def getAll[T : Manifest](
+ client: Client,
+ builder: SearchRequestBuilder)(
+ implicit formats: Formats): Seq[T] = {
+ val results = ArrayBuffer[T]()
+ var response = builder.setScroll(scrollLife).get
+ var hits = response.getHits().hits()
+ results ++= hits.map(h => read[T](h.getSourceAsString))
+ while (hits.size > 0) {
+ response = client.prepareSearchScroll(response.getScrollId).
+ setScroll(scrollLife).get
+ hits = response.getHits().hits()
+ results ++= hits.map(h => read[T](h.getSourceAsString))
+ }
+ results
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
new file mode 100644
index 0000000..75ac2b0
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.elasticsearch
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+import org.elasticsearch.client.transport.TransportClient
+import org.elasticsearch.common.settings.ImmutableSettings
+import org.elasticsearch.common.transport.InetSocketTransportAddress
+import org.elasticsearch.transport.ConnectTransportException
+
+class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
+ with Logging {
+ override val prefix = "ES"
+ val client = try {
+ val hosts = config.properties.get("HOSTS").
+ map(_.split(",").toSeq).getOrElse(Seq("localhost"))
+ val ports = config.properties.get("PORTS").
+ map(_.split(",").toSeq.map(_.toInt)).getOrElse(Seq(9300))
+ val settings = ImmutableSettings.settingsBuilder()
+ .put("cluster.name", config.properties.getOrElse("CLUSTERNAME", "elasticsearch"))
+ val transportClient = new TransportClient(settings)
+ (hosts zip ports) foreach { hp =>
+ transportClient.addTransportAddress(
+ new InetSocketTransportAddress(hp._1, hp._2))
+ }
+ transportClient
+ } catch {
+ case e: ConnectTransportException =>
+ throw new StorageClientException(e.getMessage, e)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
new file mode 100644
index 0000000..0c549b8
--- /dev/null
+++ b/storage/elasticsearch1/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** Elasticsearch implementation of storage traits, supporting meta data only
+ *
+ * @group Implementation
+ */
+package object elasticsearch {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/elasticsearch1/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/elasticsearch1/src/test/resources/application.conf b/storage/elasticsearch1/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/elasticsearch1/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/.gitignore
----------------------------------------------------------------------
diff --git a/storage/hbase/.gitignore b/storage/hbase/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/hbase/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hbase/build.sbt b/storage/hbase/build.sbt
new file mode 100644
index 0000000..5856a5e
--- /dev/null
+++ b/storage/hbase/build.sbt
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-hbase"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
+ "org.apache.hbase" % "hbase-common" % "0.98.5-hadoop2",
+ "org.apache.hbase" % "hbase-client" % "0.98.5-hadoop2"
+ exclude("org.apache.zookeeper", "zookeeper"),
+ // added for Parallel storage interface
+ "org.apache.hbase" % "hbase-server" % "0.98.5-hadoop2"
+ exclude("org.apache.hbase", "hbase-client")
+ exclude("org.apache.zookeeper", "zookeeper")
+ exclude("javax.servlet", "servlet-api")
+ exclude("org.mortbay.jetty", "servlet-api-2.5")
+ exclude("org.mortbay.jetty", "jsp-api-2.1")
+ exclude("org.mortbay.jetty", "jsp-2.1"),
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hbase-assembly-" + version.value + ".jar")
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
new file mode 100644
index 0000000..2cdb734
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.EventValidation
+import org.apache.predictionio.data.storage.DataMap
+
+import org.apache.hadoop.hbase.client.Result
+import org.apache.hadoop.hbase.client.Put
+import org.apache.hadoop.hbase.client.Scan
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.filter.FilterList
+import org.apache.hadoop.hbase.filter.RegexStringComparator
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp
+import org.apache.hadoop.hbase.filter.BinaryComparator
+import org.apache.hadoop.hbase.filter.QualifierFilter
+import org.apache.hadoop.hbase.filter.SkipFilter
+
+import org.json4s.DefaultFormats
+import org.json4s.JObject
+import org.json4s.native.Serialization.{ read, write }
+
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+
+import org.apache.commons.codec.binary.Base64
+import java.security.MessageDigest
+
+import java.util.UUID
+
+/* common utility function for accessing EventsStore in HBase */
+object HBEventsUtil {
+
+ implicit val formats = DefaultFormats
+
+ def tableName(namespace: String, appId: Int, channelId: Option[Int] = None): String = {
+ channelId.map { ch =>
+ s"${namespace}:events_${appId}_${ch}"
+ }.getOrElse {
+ s"${namespace}:events_${appId}"
+ }
+ }
+
+ // column names for "e" column family
+ val colNames: Map[String, Array[Byte]] = Map(
+ "event" -> "e",
+ "entityType" -> "ety",
+ "entityId" -> "eid",
+ "targetEntityType" -> "tety",
+ "targetEntityId" -> "teid",
+ "properties" -> "p",
+ "prId" -> "prid",
+ "eventTime" -> "et",
+ "eventTimeZone" -> "etz",
+ "creationTime" -> "ct",
+ "creationTimeZone" -> "ctz"
+ ).mapValues(Bytes.toBytes(_))
+
+ def hash(entityType: String, entityId: String): Array[Byte] = {
+ val s = entityType + "-" + entityId
+ // get a new MessageDigest object each time for thread-safe
+ val md5 = MessageDigest.getInstance("MD5")
+ md5.digest(Bytes.toBytes(s))
+ }
+
+ class RowKey(
+ val b: Array[Byte]
+ ) {
+ require((b.size == 32), s"Incorrect b size: ${b.size}")
+ lazy val entityHash: Array[Byte] = b.slice(0, 16)
+ lazy val millis: Long = Bytes.toLong(b.slice(16, 24))
+ lazy val uuidLow: Long = Bytes.toLong(b.slice(24, 32))
+
+ lazy val toBytes: Array[Byte] = b
+
+ override def toString: String = {
+ Base64.encodeBase64URLSafeString(toBytes)
+ }
+ }
+
+ object RowKey {
+ def apply(
+ entityType: String,
+ entityId: String,
+ millis: Long,
+ uuidLow: Long): RowKey = {
+ // add UUID least significant bits for multiple actions at the same time
+ // (UUID's most significant bits are actually timestamp,
+ // use eventTime instead).
+ val b = hash(entityType, entityId) ++
+ Bytes.toBytes(millis) ++ Bytes.toBytes(uuidLow)
+ new RowKey(b)
+ }
+
+ // get RowKey from string representation
+ def apply(s: String): RowKey = {
+ try {
+ apply(Base64.decodeBase64(s))
+ } catch {
+ case e: Exception => throw new RowKeyException(
+ s"Failed to convert String ${s} to RowKey because ${e}", e)
+ }
+ }
+
+ def apply(b: Array[Byte]): RowKey = {
+ if (b.size != 32) {
+ val bString = b.mkString(",")
+ throw new RowKeyException(
+ s"Incorrect byte array size. Bytes: ${bString}.")
+ }
+ new RowKey(b)
+ }
+
+ }
+
+ class RowKeyException(val msg: String, val cause: Exception)
+ extends Exception(msg, cause) {
+ def this(msg: String) = this(msg, null)
+ }
+
+ case class PartialRowKey(entityType: String, entityId: String,
+ millis: Option[Long] = None) {
+ val toBytes: Array[Byte] = {
+ hash(entityType, entityId) ++
+ (millis.map(Bytes.toBytes(_)).getOrElse(Array[Byte]()))
+ }
+ }
+
+ def eventToPut(event: Event, appId: Int): (Put, RowKey) = {
+ // generate new rowKey if eventId is None
+ val rowKey = event.eventId.map { id =>
+ RowKey(id) // create rowKey from eventId
+ }.getOrElse {
+ // TOOD: use real UUID. not pseudo random
+ val uuidLow: Long = UUID.randomUUID().getLeastSignificantBits
+ RowKey(
+ entityType = event.entityType,
+ entityId = event.entityId,
+ millis = event.eventTime.getMillis,
+ uuidLow = uuidLow
+ )
+ }
+
+ val eBytes = Bytes.toBytes("e")
+ // use eventTime as HBase's cell timestamp
+ val put = new Put(rowKey.toBytes, event.eventTime.getMillis)
+
+ def addStringToE(col: Array[Byte], v: String): Put = {
+ put.add(eBytes, col, Bytes.toBytes(v))
+ }
+
+ def addLongToE(col: Array[Byte], v: Long): Put = {
+ put.add(eBytes, col, Bytes.toBytes(v))
+ }
+
+ addStringToE(colNames("event"), event.event)
+ addStringToE(colNames("entityType"), event.entityType)
+ addStringToE(colNames("entityId"), event.entityId)
+
+ event.targetEntityType.foreach { targetEntityType =>
+ addStringToE(colNames("targetEntityType"), targetEntityType)
+ }
+
+ event.targetEntityId.foreach { targetEntityId =>
+ addStringToE(colNames("targetEntityId"), targetEntityId)
+ }
+
+ // TODO: make properties Option[]
+ if (!event.properties.isEmpty) {
+ addStringToE(colNames("properties"), write(event.properties.toJObject))
+ }
+
+ event.prId.foreach { prId =>
+ addStringToE(colNames("prId"), prId)
+ }
+
+ addLongToE(colNames("eventTime"), event.eventTime.getMillis)
+ val eventTimeZone = event.eventTime.getZone
+ if (!eventTimeZone.equals(EventValidation.defaultTimeZone)) {
+ addStringToE(colNames("eventTimeZone"), eventTimeZone.getID)
+ }
+
+ addLongToE(colNames("creationTime"), event.creationTime.getMillis)
+ val creationTimeZone = event.creationTime.getZone
+ if (!creationTimeZone.equals(EventValidation.defaultTimeZone)) {
+ addStringToE(colNames("creationTimeZone"), creationTimeZone.getID)
+ }
+
+ // can use zero-length byte array for tag cell value
+ (put, rowKey)
+ }
+
+ def resultToEvent(result: Result, appId: Int): Event = {
+ val rowKey = RowKey(result.getRow())
+
+ val eBytes = Bytes.toBytes("e")
+ // val e = result.getFamilyMap(eBytes)
+
+ def getStringCol(col: String): String = {
+ val r = result.getValue(eBytes, colNames(col))
+ require(r != null,
+ s"Failed to get value for column ${col}. " +
+ s"Rowkey: ${rowKey.toString} " +
+ s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
+
+ Bytes.toString(r)
+ }
+
+ def getLongCol(col: String): Long = {
+ val r = result.getValue(eBytes, colNames(col))
+ require(r != null,
+ s"Failed to get value for column ${col}. " +
+ s"Rowkey: ${rowKey.toString} " +
+ s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
+
+ Bytes.toLong(r)
+ }
+
+ def getOptStringCol(col: String): Option[String] = {
+ val r = result.getValue(eBytes, colNames(col))
+ if (r == null) {
+ None
+ } else {
+ Some(Bytes.toString(r))
+ }
+ }
+
+ def getTimestamp(col: String): Long = {
+ result.getColumnLatestCell(eBytes, colNames(col)).getTimestamp()
+ }
+
+ val event = getStringCol("event")
+ val entityType = getStringCol("entityType")
+ val entityId = getStringCol("entityId")
+ val targetEntityType = getOptStringCol("targetEntityType")
+ val targetEntityId = getOptStringCol("targetEntityId")
+ val properties: DataMap = getOptStringCol("properties")
+ .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
+ val prId = getOptStringCol("prId")
+ val eventTimeZone = getOptStringCol("eventTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+ val eventTime = new DateTime(
+ getLongCol("eventTime"), eventTimeZone)
+ val creationTimeZone = getOptStringCol("creationTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+ val creationTime: DateTime = new DateTime(
+ getLongCol("creationTime"), creationTimeZone)
+
+ Event(
+ eventId = Some(RowKey(result.getRow()).toString),
+ event = event,
+ entityType = entityType,
+ entityId = entityId,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ properties = properties,
+ eventTime = eventTime,
+ tags = Seq(),
+ prId = prId,
+ creationTime = creationTime
+ )
+ }
+
+
+ // for mandatory field. None means don't care.
+ // for optional field. None means don't care.
+ // Some(None) means not exist.
+ // Some(Some(x)) means it should match x
+ def createScan(
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None,
+ reversed: Option[Boolean] = None): Scan = {
+
+ val scan: Scan = new Scan()
+
+ (entityType, entityId) match {
+ case (Some(et), Some(eid)) => {
+ val start = PartialRowKey(et, eid,
+ startTime.map(_.getMillis)).toBytes
+ // if no untilTime, stop when reach next bytes of entityTypeAndId
+ val stop = PartialRowKey(et, eid,
+ untilTime.map(_.getMillis).orElse(Some(-1))).toBytes
+
+ if (reversed.getOrElse(false)) {
+ // Reversed order.
+ // If you specify a startRow and stopRow,
+ // to scan in reverse, the startRow needs to be lexicographically
+ // after the stopRow.
+ scan.setStartRow(stop)
+ scan.setStopRow(start)
+ scan.setReversed(true)
+ } else {
+ scan.setStartRow(start)
+ scan.setStopRow(stop)
+ }
+ }
+ case (_, _) => {
+ val minTime: Long = startTime.map(_.getMillis).getOrElse(0)
+ val maxTime: Long = untilTime.map(_.getMillis).getOrElse(Long.MaxValue)
+ scan.setTimeRange(minTime, maxTime)
+ if (reversed.getOrElse(false)) {
+ scan.setReversed(true)
+ }
+ }
+ }
+
+ val filters = new FilterList(FilterList.Operator.MUST_PASS_ALL)
+
+ val eBytes = Bytes.toBytes("e")
+
+ def createBinaryFilter(col: String, value: Array[Byte]): SingleColumnValueFilter = {
+ val comp = new BinaryComparator(value)
+ new SingleColumnValueFilter(
+ eBytes, colNames(col), CompareOp.EQUAL, comp)
+ }
+
+ // skip the row if the column exists
+ def createSkipRowIfColumnExistFilter(col: String): SkipFilter = {
+ val comp = new BinaryComparator(colNames(col))
+ val q = new QualifierFilter(CompareOp.NOT_EQUAL, comp)
+ // filters an entire row if any of the Cell checks do not pass
+ new SkipFilter(q)
+ }
+
+ entityType.foreach { et =>
+ val compType = new BinaryComparator(Bytes.toBytes(et))
+ val filterType = new SingleColumnValueFilter(
+ eBytes, colNames("entityType"), CompareOp.EQUAL, compType)
+ filters.addFilter(filterType)
+ }
+
+ entityId.foreach { eid =>
+ val compId = new BinaryComparator(Bytes.toBytes(eid))
+ val filterId = new SingleColumnValueFilter(
+ eBytes, colNames("entityId"), CompareOp.EQUAL, compId)
+ filters.addFilter(filterId)
+ }
+
+ eventNames.foreach { eventsList =>
+ // match any of event in the eventsList
+ val eventFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE)
+ eventsList.foreach { e =>
+ val compEvent = new BinaryComparator(Bytes.toBytes(e))
+ val filterEvent = new SingleColumnValueFilter(
+ eBytes, colNames("event"), CompareOp.EQUAL, compEvent)
+ eventFilters.addFilter(filterEvent)
+ }
+ if (!eventFilters.getFilters().isEmpty) {
+ filters.addFilter(eventFilters)
+ }
+ }
+
+ targetEntityType.foreach { tetOpt =>
+ if (tetOpt.isEmpty) {
+ val filter = createSkipRowIfColumnExistFilter("targetEntityType")
+ filters.addFilter(filter)
+ } else {
+ tetOpt.foreach { tet =>
+ val filter = createBinaryFilter(
+ "targetEntityType", Bytes.toBytes(tet))
+ // the entire row will be skipped if the column is not found.
+ filter.setFilterIfMissing(true)
+ filters.addFilter(filter)
+ }
+ }
+ }
+
+ targetEntityId.foreach { teidOpt =>
+ if (teidOpt.isEmpty) {
+ val filter = createSkipRowIfColumnExistFilter("targetEntityId")
+ filters.addFilter(filter)
+ } else {
+ teidOpt.foreach { teid =>
+ val filter = createBinaryFilter(
+ "targetEntityId", Bytes.toBytes(teid))
+ // the entire row will be skipped if the column is not found.
+ filter.setFilterIfMissing(true)
+ filters.addFilter(filter)
+ }
+ }
+ }
+
+ if (!filters.getFilters().isEmpty) {
+ scan.setFilter(filters)
+ }
+
+ scan
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
new file mode 100644
index 0000000..360b007
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.LEvents
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.hbase.HBEventsUtil.RowKey
+import org.apache.hadoop.hbase.HColumnDescriptor
+import org.apache.hadoop.hbase.HTableDescriptor
+import org.apache.hadoop.hbase.NamespaceDescriptor
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.client._
+import org.joda.time.DateTime
+
+import scala.collection.JavaConversions._
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+class HBLEvents(val client: HBClient, config: StorageClientConfig, val namespace: String)
+ extends LEvents with Logging {
+
+ // implicit val formats = DefaultFormats + new EventJson4sSupport.DBSerializer
+
+ def resultToEvent(result: Result, appId: Int): Event =
+ HBEventsUtil.resultToEvent(result, appId)
+
+ def getTable(appId: Int, channelId: Option[Int] = None): HTableInterface =
+ client.connection.getTable(HBEventsUtil.tableName(namespace, appId, channelId))
+
+ override
+ def init(appId: Int, channelId: Option[Int] = None): Boolean = {
+ // check namespace exist
+ val existingNamespace = client.admin.listNamespaceDescriptors()
+ .map(_.getName)
+ if (!existingNamespace.contains(namespace)) {
+ val nameDesc = NamespaceDescriptor.create(namespace).build()
+ info(s"The namespace ${namespace} doesn't exist yet. Creating now...")
+ client.admin.createNamespace(nameDesc)
+ }
+
+ val tableName = TableName.valueOf(HBEventsUtil.tableName(namespace, appId, channelId))
+ if (!client.admin.tableExists(tableName)) {
+ info(s"The table ${tableName.getNameAsString()} doesn't exist yet." +
+ " Creating now...")
+ val tableDesc = new HTableDescriptor(tableName)
+ tableDesc.addFamily(new HColumnDescriptor("e"))
+ tableDesc.addFamily(new HColumnDescriptor("r")) // reserved
+ client.admin.createTable(tableDesc)
+ }
+ true
+ }
+
+ override
+ def remove(appId: Int, channelId: Option[Int] = None): Boolean = {
+ val tableName = TableName.valueOf(HBEventsUtil.tableName(namespace, appId, channelId))
+ try {
+ if (client.admin.tableExists(tableName)) {
+ info(s"Removing table ${tableName.getNameAsString()}...")
+ client.admin.disableTable(tableName)
+ client.admin.deleteTable(tableName)
+ } else {
+ info(s"Table ${tableName.getNameAsString()} doesn't exist." +
+ s" Nothing is deleted.")
+ }
+ true
+ } catch {
+ case e: Exception => {
+ error(s"Fail to remove table for appId ${appId}. Exception: ${e}")
+ false
+ }
+ }
+ }
+
+ override
+ def close(): Unit = {
+ client.admin.close()
+ client.connection.close()
+ }
+
+ override
+ def futureInsert(
+ event: Event, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
+ Future[String] = {
+ Future {
+ val table = getTable(appId, channelId)
+ val (put, rowKey) = HBEventsUtil.eventToPut(event, appId)
+ table.put(put)
+ table.flushCommits()
+ table.close()
+ rowKey.toString
+ }
+ }
+
+ override
+ def futureGet(
+ eventId: String, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
+ Future[Option[Event]] = {
+ Future {
+ val table = getTable(appId, channelId)
+ val rowKey = RowKey(eventId)
+ val get = new Get(rowKey.toBytes)
+
+ val result = table.get(get)
+ table.close()
+
+ if (!result.isEmpty()) {
+ val event = resultToEvent(result, appId)
+ Some(event)
+ } else {
+ None
+ }
+ }
+ }
+
+ override
+ def futureDelete(
+ eventId: String, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
+ Future[Boolean] = {
+ Future {
+ val table = getTable(appId, channelId)
+ val rowKey = RowKey(eventId)
+ val exists = table.exists(new Get(rowKey.toBytes))
+ table.delete(new Delete(rowKey.toBytes))
+ table.close()
+ exists
+ }
+ }
+
+ override
+ def futureFind(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None,
+ limit: Option[Int] = None,
+ reversed: Option[Boolean] = None)(implicit ec: ExecutionContext):
+ Future[Iterator[Event]] = {
+ Future {
+
+ require(!((reversed == Some(true)) && (entityType.isEmpty || entityId.isEmpty)),
+ "the parameter reversed can only be used with both entityType and entityId specified.")
+
+ val table = getTable(appId, channelId)
+
+ val scan = HBEventsUtil.createScan(
+ startTime = startTime,
+ untilTime = untilTime,
+ entityType = entityType,
+ entityId = entityId,
+ eventNames = eventNames,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ reversed = reversed)
+ val scanner = table.getScanner(scan)
+ table.close()
+
+ val eventsIter = scanner.iterator()
+
+ // Get all events if None or Some(-1)
+ val results: Iterator[Result] = limit match {
+ case Some(-1) => eventsIter
+ case None => eventsIter
+ case Some(x) => eventsIter.take(x)
+ }
+
+ val eventsIt = results.map { resultToEvent(_, appId) }
+
+ eventsIt
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
new file mode 100644
index 0000000..7324fa6
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.client.{Delete, HTable, Result}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.mapreduce.{TableInputFormat, TableOutputFormat}
+import org.apache.hadoop.io.Writable
+import org.apache.hadoop.mapreduce.OutputFormat
+import org.apache.predictionio.data.storage.{Event, PEvents, StorageClientConfig}
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.joda.time.DateTime
+
+class HBPEvents(client: HBClient, config: StorageClientConfig, namespace: String) extends PEvents {
+
+ def checkTableExists(appId: Int, channelId: Option[Int]): Unit = {
+ if (!client.admin.tableExists(HBEventsUtil.tableName(namespace, appId, channelId))) {
+ if (channelId.nonEmpty) {
+ logger.error(s"The appId $appId with channelId $channelId does not exist." +
+ s" Please use valid appId and channelId.")
+ throw new Exception(s"HBase table not found for appId $appId" +
+ s" with channelId $channelId.")
+ } else {
+ logger.error(s"The appId $appId does not exist. Please use valid appId.")
+ throw new Exception(s"HBase table not found for appId $appId.")
+ }
+ }
+ }
+
+ override
+ def find(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None
+ )(sc: SparkContext): RDD[Event] = {
+
+ checkTableExists(appId, channelId)
+
+ val conf = HBaseConfiguration.create()
+ conf.set(TableInputFormat.INPUT_TABLE,
+ HBEventsUtil.tableName(namespace, appId, channelId))
+
+ val scan = HBEventsUtil.createScan(
+ startTime = startTime,
+ untilTime = untilTime,
+ entityType = entityType,
+ entityId = entityId,
+ eventNames = eventNames,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ reversed = None)
+ scan.setCaching(500) // TODO
+ scan.setCacheBlocks(false) // TODO
+
+ conf.set(TableInputFormat.SCAN, PIOHBaseUtil.convertScanToString(scan))
+
+ // HBase is not accessed until this rdd is actually used.
+ val rdd = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
+ classOf[ImmutableBytesWritable],
+ classOf[Result]).map {
+ case (key, row) => HBEventsUtil.resultToEvent(row, appId)
+ }
+
+ rdd
+ }
+
+ override
+ def write(
+ events: RDD[Event], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+
+ checkTableExists(appId, channelId)
+
+ val conf = HBaseConfiguration.create()
+ conf.set(TableOutputFormat.OUTPUT_TABLE,
+ HBEventsUtil.tableName(namespace, appId, channelId))
+ conf.setClass("mapreduce.outputformat.class",
+ classOf[TableOutputFormat[Object]],
+ classOf[OutputFormat[Object, Writable]])
+
+ events.map { event =>
+ val (put, rowKey) = HBEventsUtil.eventToPut(event, appId)
+ (new ImmutableBytesWritable(rowKey.toBytes), put)
+ }.saveAsNewAPIHadoopDataset(conf)
+
+ }
+
+ def delete(
+ eventIds: RDD[String], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+
+ checkTableExists(appId, channelId)
+
+ val tableName = HBEventsUtil.tableName(namespace, appId, channelId)
+
+ eventIds.foreachPartition{ iter =>
+ val conf = HBaseConfiguration.create()
+ conf.set(TableOutputFormat.OUTPUT_TABLE,
+ tableName)
+
+ val table = new HTable(conf, tableName)
+ iter.foreach { id =>
+ val rowKey = HBEventsUtil.RowKey(id)
+ val delete = new Delete(rowKey.b)
+ table.delete(delete)
+ }
+ table.close
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
new file mode 100644
index 0000000..745fcb9
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.hadoop.hbase.client.Scan
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil
+import org.apache.hadoop.hbase.util.Base64
+
+object PIOHBaseUtil {
+ /*
+ * Copying this from Apache HBase because of its restrictive scope in 0.98.x
+ */
+ def convertScanToString(scan: Scan): String = {
+ val proto = ProtobufUtil.toScan(scan)
+ Base64.encodeBytes(proto.toByteArray)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
new file mode 100644
index 0000000..1720410
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase
+
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.MasterNotRunningException
+import org.apache.hadoop.hbase.ZooKeeperConnectionException
+import org.apache.hadoop.hbase.client.HConnectionManager
+import org.apache.hadoop.hbase.client.HConnection
+import org.apache.hadoop.hbase.client.HBaseAdmin
+
+import grizzled.slf4j.Logging
+
+case class HBClient(
+ val conf: Configuration,
+ val connection: HConnection,
+ val admin: HBaseAdmin
+)
+
+class StorageClient(val config: StorageClientConfig)
+ extends BaseStorageClient with Logging {
+
+ val conf = HBaseConfiguration.create()
+
+ if (config.test) {
+ // use fewer retries and shorter timeout for test mode
+ conf.set("hbase.client.retries.number", "1")
+ conf.set("zookeeper.session.timeout", "30000");
+ conf.set("zookeeper.recovery.retry", "1")
+ }
+
+ try {
+ HBaseAdmin.checkHBaseAvailable(conf)
+ } catch {
+ case e: MasterNotRunningException =>
+ error("HBase master is not running (ZooKeeper ensemble: " +
+ conf.get("hbase.zookeeper.quorum") + "). Please make sure that HBase " +
+ "is running properly, and that the configuration is pointing at the " +
+ "correct ZooKeeper ensemble.")
+ throw e
+ case e: ZooKeeperConnectionException =>
+ error("Cannot connect to ZooKeeper (ZooKeeper ensemble: " +
+ conf.get("hbase.zookeeper.quorum") + "). Please make sure that the " +
+ "configuration is pointing at the correct ZooKeeper ensemble. By " +
+ "default, HBase manages its own ZooKeeper, so if you have not " +
+ "configured HBase to use an external ZooKeeper, that means your " +
+ "HBase is not started or configured properly.")
+ throw e
+ case e: Exception => {
+ error("Failed to connect to HBase." +
+ " Please check if HBase is running properly.")
+ throw e
+ }
+ }
+
+ val connection = HConnectionManager.createConnection(conf)
+
+ val client = HBClient(
+ conf = conf,
+ connection = connection,
+ admin = new HBaseAdmin(connection)
+ )
+
+ override
+ val prefix = "HB"
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
new file mode 100644
index 0000000..49bf031
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** HBase implementation of storage traits, supporting event data only
+ *
+ * @group Implementation
+ */
+package object hbase {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
new file mode 100644
index 0000000..cc07fa4
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/HB_0_8_0.scala
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase.upgrade
+
+import org.apache.predictionio.annotation.Experimental
+
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.EventValidation
+import org.apache.predictionio.data.storage.DataMap
+
+import org.apache.hadoop.hbase.client.Scan
+import org.apache.hadoop.hbase.client.HConnection
+import org.apache.hadoop.hbase.client.Result
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.util.Bytes
+
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+
+import org.json4s.DefaultFormats
+import org.json4s.JObject
+import org.json4s.native.Serialization.{ read, write }
+
+import org.apache.commons.codec.binary.Base64
+
+import scala.collection.JavaConversions._
+
+/** :: Experimental :: */
+@Experimental
+object HB_0_8_0 {
+
+ implicit val formats = DefaultFormats
+
+ def getByAppId(
+ connection: HConnection,
+ namespace: String,
+ appId: Int): Iterator[Event] = {
+ val tableName = TableName.valueOf(namespace, "events")
+ val table = connection.getTable(tableName)
+ val start = PartialRowKey(appId)
+ val stop = PartialRowKey(appId + 1)
+ val scan = new Scan(start.toBytes, stop.toBytes)
+ val scanner = table.getScanner(scan)
+ table.close()
+ scanner.iterator().map { resultToEvent(_) }
+ }
+
+ val colNames: Map[String, Array[Byte]] = Map(
+ "event" -> "e",
+ "entityType" -> "ety",
+ "entityId" -> "eid",
+ "targetEntityType" -> "tety",
+ "targetEntityId" -> "teid",
+ "properties" -> "p",
+ "prId" -> "pk", // columna name is 'pk' in 0.8.0/0.8.1
+ "eventTimeZone" -> "etz",
+ "creationTimeZone" -> "ctz"
+ ).mapValues(Bytes.toBytes(_))
+
+
+ class RowKey(
+ val appId: Int,
+ val millis: Long,
+ val uuidLow: Long
+ ) {
+ lazy val toBytes: Array[Byte] = {
+ // add UUID least significant bits for multiple actions at the same time
+ // (UUID's most significant bits are actually timestamp,
+ // use eventTime instead).
+ Bytes.toBytes(appId) ++ Bytes.toBytes(millis) ++ Bytes.toBytes(uuidLow)
+ }
+ override def toString: String = {
+ Base64.encodeBase64URLSafeString(toBytes)
+ }
+ }
+
+ object RowKey {
+ // get RowKey from string representation
+ def apply(s: String): RowKey = {
+ try {
+ apply(Base64.decodeBase64(s))
+ } catch {
+ case e: Exception => throw new RowKeyException(
+ s"Failed to convert String ${s} to RowKey because ${e}", e)
+ }
+ }
+
+ def apply(b: Array[Byte]): RowKey = {
+ if (b.size != 20) {
+ val bString = b.mkString(",")
+ throw new RowKeyException(
+ s"Incorrect byte array size. Bytes: ${bString}.")
+ }
+
+ new RowKey(
+ appId = Bytes.toInt(b.slice(0, 4)),
+ millis = Bytes.toLong(b.slice(4, 12)),
+ uuidLow = Bytes.toLong(b.slice(12, 20))
+ )
+ }
+ }
+
+ class RowKeyException(msg: String, cause: Exception)
+ extends Exception(msg, cause) {
+ def this(msg: String) = this(msg, null)
+ }
+
+ case class PartialRowKey(val appId: Int, val millis: Option[Long] = None) {
+ val toBytes: Array[Byte] = {
+ Bytes.toBytes(appId) ++
+ (millis.map(Bytes.toBytes(_)).getOrElse(Array[Byte]()))
+ }
+ }
+
+ def resultToEvent(result: Result): Event = {
+ val rowKey = RowKey(result.getRow())
+
+ val eBytes = Bytes.toBytes("e")
+ // val e = result.getFamilyMap(eBytes)
+
+ def getStringCol(col: String): String = {
+ val r = result.getValue(eBytes, colNames(col))
+ require(r != null,
+ s"Failed to get value for column ${col}. " +
+ s"Rowkey: ${rowKey.toString} " +
+ s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
+
+ Bytes.toString(r)
+ }
+
+ def getOptStringCol(col: String): Option[String] = {
+ val r = result.getValue(eBytes, colNames(col))
+ if (r == null) {
+ None
+ } else {
+ Some(Bytes.toString(r))
+ }
+ }
+
+ def getTimestamp(col: String): Long = {
+ result.getColumnLatestCell(eBytes, colNames(col)).getTimestamp()
+ }
+
+ val event = getStringCol("event")
+ val entityType = getStringCol("entityType")
+ val entityId = getStringCol("entityId")
+ val targetEntityType = getOptStringCol("targetEntityType")
+ val targetEntityId = getOptStringCol("targetEntityId")
+ val properties: DataMap = getOptStringCol("properties")
+ .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
+ val prId = getOptStringCol("prId")
+ val eventTimeZone = getOptStringCol("eventTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+ val creationTimeZone = getOptStringCol("creationTimeZone")
+ .map(DateTimeZone.forID(_))
+ .getOrElse(EventValidation.defaultTimeZone)
+
+ val creationTime: DateTime = new DateTime(
+ getTimestamp("event"), creationTimeZone
+ )
+
+ Event(
+ eventId = Some(RowKey(result.getRow()).toString),
+ event = event,
+ entityType = entityType,
+ entityId = entityId,
+ targetEntityType = targetEntityType,
+ targetEntityId = targetEntityId,
+ properties = properties,
+ eventTime = new DateTime(rowKey.millis, eventTimeZone),
+ tags = Seq(),
+ prId = prId,
+ creationTime = creationTime
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
new file mode 100644
index 0000000..1759561
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase.upgrade
+
+import org.apache.predictionio.annotation.Experimental
+
+import org.apache.predictionio.data.storage.Storage
+import org.apache.predictionio.data.storage.hbase.HBLEvents
+import org.apache.predictionio.data.storage.hbase.HBEventsUtil
+
+import scala.collection.JavaConversions._
+
+/** :: Experimental :: */
+@Experimental
+object Upgrade {
+
+ def main(args: Array[String]) {
+ val fromAppId = args(0).toInt
+ val toAppId = args(1).toInt
+ val batchSize = args.lift(2).map(_.toInt).getOrElse(100)
+ val fromNamespace = args.lift(3).getOrElse("predictionio_eventdata")
+
+ upgrade(fromAppId, toAppId, batchSize, fromNamespace)
+ }
+
+ /* For upgrade from 0.8.0 or 0.8.1 to 0.8.2 only */
+ def upgrade(
+ fromAppId: Int,
+ toAppId: Int,
+ batchSize: Int,
+ fromNamespace: String) {
+
+ val events = Storage.getLEvents().asInstanceOf[HBLEvents]
+
+ // Assume already run "pio app new <newapp>" (new app already created)
+ // TODO: check if new table empty and warn user if not
+ val newTable = events.getTable(toAppId)
+
+ val newTableName = newTable.getName().getNameAsString()
+ println(s"Copying data from ${fromNamespace}:events for app ID ${fromAppId}"
+ + s" to new HBase table ${newTableName}...")
+
+ HB_0_8_0.getByAppId(
+ events.client.connection,
+ fromNamespace,
+ fromAppId).grouped(batchSize).foreach { eventGroup =>
+ val puts = eventGroup.map{ e =>
+ val (put, rowkey) = HBEventsUtil.eventToPut(e, toAppId)
+ put
+ }
+ newTable.put(puts.toList)
+ }
+
+ newTable.flushCommits()
+ newTable.close()
+ println("Done.")
+ }
+
+}
[37/50] [abbrv] incubator-predictionio git commit: Documentation
polish
Posted by do...@apache.org.
Documentation polish
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/bea82a6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/bea82a6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/bea82a6c
Branch: refs/heads/livedoc
Commit: bea82a6c3d14e587ce54738896666bd1a577a9c5
Parents: 23a8693
Author: Donald Szeto <do...@apache.org>
Authored: Thu Mar 30 11:26:26 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Thu Mar 30 11:26:26 2017 -0700
----------------------------------------------------------------------
RELEASE.md | 74 ++++++++++++++++----
bin/install.sh | 8 +--
conf/pio-env.sh.template | 4 +-
.../predictionio/data/storage/Storage.scala | 42 +++++++++--
docs/manual/config.rb | 4 +-
docs/manual/data/versions.yml | 11 +--
.../source/community/contribute-code.html.md | 2 +-
.../source/community/contribute-sdk.html.md | 6 +-
docs/manual/source/customize/dase.html.md.erb | 18 ++---
.../source/datacollection/eventapi.html.md | 4 +-
.../datacollection/eventmodel.html.md.erb | 4 +-
docs/manual/source/demo/tapster.html.md | 4 +-
.../source/demo/textclassification.html.md.erb | 8 +--
docs/manual/source/gallery/templates.yaml | 7 +-
docs/manual/source/install/index.html.md.erb | 18 +----
.../install/install-sourcecode.html.md.erb | 32 +++++++--
.../source/install/install-vagrant.html.md.erb | 2 +-
.../source/partials/_action_call.html.slim | 2 +-
docs/manual/source/partials/_footer.html.slim | 4 +-
.../shared/install/_elasticsearch.html.erb | 9 ++-
.../partials/shared/install/_hbase.html.erb | 10 ++-
.../partials/shared/install/_postgres.html.erb | 23 ++++--
docs/manual/source/resources/faq.html.md | 48 ++++++++-----
docs/manual/source/resources/upgrade.html.md | 19 +++--
docs/manual/source/sdk/java.html.md.erb | 2 +-
docs/manual/source/sdk/php.html.md.erb | 2 +-
docs/manual/source/sdk/ruby.html.md.erb | 2 +-
docs/manual/source/tryit/index.html.slim | 2 +-
tests/docker-files/env-conf/pio-env.sh | 2 +-
.../engines/recommendation-engine/README.md | 2 +-
tests/pio_tests/scenarios/eventserver_test.py | 4 +-
tests/pio_tests/utils.py | 2 +-
.../tools/commands/Management.scala | 9 ++-
33 files changed, 258 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/RELEASE.md
----------------------------------------------------------------------
diff --git a/RELEASE.md b/RELEASE.md
index 885faa3..baa5246 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -15,15 +15,61 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-#Release Notes and News
+# Release Notes and News
**Note:** For upgrade instructions please refer to [this page](http://predictionio.incubator.apache.org/resources/upgrade/).
## Version History
+### 0.11.0
+
+Apr ?, 2017
+
+#### New Features
+
+- PIO-30: Scala 2.11 support
+- PIO-30: Spark 2 support
+- PIO-49: Elasticsearch 5 support
+- PIO-30, PIO-49: Flexible build system
+- PIO-47, PIO-51: Removal of engine manifests
+- PIO-49: Modularized storage backend modules
+- PIO-45: Self cleaning data source
+
+#### Behavior Changes
+
+- PIO-25: `pio-start-all` will no longer start PostgreSQL if it is not being
+ used.
+- PIO-47, PIO-51: `pio build` no longer requires access to the metadata
+ repository. `pio` commands will now accept an optional `--engine-dir`
+ parameter if you want to run `pio build`, `pio train`, or `pio deploy` outside
+ of an engine directory. This is an interim solution before an engine registry
+ feature becomes available in the future.
+- PIO-54: Generated access keys will no longer start with a `-` character.
+
+#### Other Changes
+
+- PIO-28: Code refactoring of the command line interface. It is now possible to
+ develop new interfaces that perform the same functionalities provided by the
+ CLI.
+- PIO-53: Integration tests can now be tied to every single Git commit, without
+ the need to update the official test Docker image.
+- The meta data and model data access object methods are now public and marked
+ as Core Developer API.
+
+#### Credits
+
+The following contributors have spent a great deal of effort to bring to you
+this release:
+
+Ahmet DAL, Alexander Merritt, Bansari Shah, Chan Lee, Chris Woodford, Daniel
+Gabrieli, Dennis Jung, Donald Szeto, Emily Rose, Hari Charan Ayada,
+infoquestsolutions, Jonny Daenen, Kenneth Chan, Laertis Pappas, Marcin
+Ziemi\u0144ski, Naoki Takezoe, Rajdeep Dua, Shinsuke Sugaya, Pat Ferrel, scorpiovn,
+Suneel Marthi, Steven Yan, Takahiro Hagino, Takako Shimamoto
+
### 0.10.0
-Aug ??, 2016
+Oct 7, 2016
- Make SSL optional
- Merge ActionML fork
@@ -36,10 +82,10 @@ Aug 5, 2016
- changed version id so artifacts don't conflict with naming in the Salesforce sponsored project.
- bug fix in memory use during moving window event trim and compaction EventStore data.
- update [install.sh](https://github.com/actionml/PredictionIO/blob/master/bin/install.sh) script for single line installs with options that support various combinations required by some templates.
-
-### v0.9.6
-April 11, 2015
+### 0.9.6
+
+April 11, 2015
For a detailed list of commits check [this page](https://github.com/apache/incubator-predictionio/commits/master)
@@ -47,7 +93,7 @@ For a detailed list of commits check [this page](https://github.com/apache/incub
- Support for maintaining a moving window of events by discarding old events from the EventStore
- Support for doing a deploy without creating a Spark Context
-### v0.9.6 (ActionML fork)
+### 0.9.6 (ActionML fork)
March 26, 2016
@@ -55,9 +101,9 @@ March 26, 2016
- Support for maintaining a moving window of events by discarding old events from the EventStore
- Support for doing a deploy without creating a Spark Context
-###v0.9.5
+### 0.9.5
-October 14th, 2015
+October 14th, 2015
[Release Notes](https://github.com/apache/incubator-predictionio/blob/master/RELEASE.md) have been moved to Github and you are reading them. For a detailed list of commits check [this page](https://github.com/apache/incubator-predictionio/commits/v0.9.5)
@@ -65,7 +111,7 @@ October 14th, 2015
- Support creating an Elasticsearch StorageClient created for an Elasticsearch cluster from variables in pio-env.sh
- Fixed some errors installing PredictionIO through install.sh when SBT was not correctly downloaded
-###v0.9.4
+### 0.9.4
July 16th, 2015
@@ -76,7 +122,7 @@ Release Notes
- Support running `pio eval` without `engine.json`
- Fix an issue where `--verbose` is not handled properly by `pio train`
-###v0.9.3
+### 0.9.3
May 20th, 2015
@@ -89,7 +135,7 @@ Release Notes
- Creation of specific app access keys
- Prevent a case where `pio build` accidentally removes PredictionIO core library
-###v0.9.2
+### 0.9.2
April 14th, 2015
@@ -101,7 +147,7 @@ Release Notes
- Engine and Event Servers now by default bind to 0.0.0.0
- Many documentation improvements
-###v0.9.1
+### 0.9.1
March 17th, 2015
@@ -110,7 +156,7 @@ Release Notes
- Improved `pio-start-all`
- Fixed a bug where `pio build` failed to set PredictionIO dependency version for engine templates
-###v0.9.0
+### 0.9.0
March 4th, 2015
@@ -121,7 +167,7 @@ Release Notes
- [Lead Scoring Template](http://predictionio.incubator.apache.org/gallery/template-gallery#classification) predicts the probability of an user will convert in the current session
- `pio-start-all`, `pio-stop-all` commands to start and stop all PredictionIO related services
-###v0.8.6
+### 0.8.6
Feb 10th, 2015
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/bin/install.sh
----------------------------------------------------------------------
diff --git a/bin/install.sh b/bin/install.sh
index 7431b09..ca7fcac 100755
--- a/bin/install.sh
+++ b/bin/install.sh
@@ -22,8 +22,8 @@ SPARK_VERSION=1.6.3
# Looks like support for Elasticsearch 2.0 will require 2.0 so deferring
ELASTICSEARCH_VERSION=1.7.5
HBASE_VERSION=1.2.2
-POSTGRES_VERSION=9.4-1204.jdbc41
-MYSQL_VERSION=5.1.37
+POSTGRES_VERSION=42.0.0
+MYSQL_VERSION=5.1.41
PIO_DIR=$HOME/PredictionIO
USER_PROFILE=$HOME/.profile
PIO_FILE=PredictionIO-*.tar.gz
@@ -284,9 +284,9 @@ files=$(ls PredictionIO*.tar.gz 2> /dev/null | wc -l)
if [[ $files == 0 ]]; then
echo "Downloading PredictionIO..."
- curl -L https://codeload.github.com/apache/incubator-predictionio/tar.gz/master > incubator-predictionio-master.tar.gz
+ curl -L https://codeload.github.com/apache/incubator-predictionio/tar.gz/master > incubator-predictionio-master.tar.gz
- tar zxf incubator-predictionio-master.tar.gz
+ tar zxf incubator-predictionio-master.tar.gz
mv incubator-predictionio-master PredictionIO
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/conf/pio-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.template b/conf/pio-env.sh.template
index 57185ba..c3a1e07 100644
--- a/conf/pio-env.sh.template
+++ b/conf/pio-env.sh.template
@@ -27,8 +27,8 @@
# SPARK_HOME=$PIO_HOME/vendors/spark-2.0.2-bin-hadoop2.7
SPARK_HOME=$PIO_HOME/vendors/spark-1.6.3-bin-hadoop2.6
-POSTGRES_JDBC_DRIVER=$PIO_HOME/lib/postgresql-9.4-1204.jdbc41.jar
-MYSQL_JDBC_DRIVER=$PIO_HOME/lib/mysql-connector-java-5.1.37.jar
+POSTGRES_JDBC_DRIVER=$PIO_HOME/lib/postgresql-42.0.0.jar
+MYSQL_JDBC_DRIVER=$PIO_HOME/lib/mysql-connector-java-5.1.41.jar
# ES_CONF_DIR: You must configure this if you have advanced configuration for
# your Elasticsearch setup.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
index 6372446..d61ad4f 100644
--- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
+++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala
@@ -362,22 +362,52 @@ object Storage extends Logging {
eventsDb.close()
}
- private[predictionio] def getMetaDataEngineInstances(): EngineInstances =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[EngineInstances]]
+ *
+ * @return An implementation of [[EngineInstances]], depending on the runtime configuration
+ */
+ def getMetaDataEngineInstances(): EngineInstances =
getDataObjectFromRepo[EngineInstances](MetaDataRepository)
- private[predictionio] def getMetaDataEvaluationInstances(): EvaluationInstances =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[EvaluationInstances]]
+ *
+ * @return An implementation of [[EvaluationInstances]], depending on the runtime configuration
+ */
+ def getMetaDataEvaluationInstances(): EvaluationInstances =
getDataObjectFromRepo[EvaluationInstances](MetaDataRepository)
- private[predictionio] def getMetaDataApps(): Apps =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[Apps]]
+ *
+ * @return An implementation of [[Apps]], depending on the runtime configuration
+ */
+ def getMetaDataApps(): Apps =
getDataObjectFromRepo[Apps](MetaDataRepository)
- private[predictionio] def getMetaDataAccessKeys(): AccessKeys =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[AccessKeys]]
+ *
+ * @return An implementation of [[AccessKeys]], depending on the runtime configuration
+ */
+ def getMetaDataAccessKeys(): AccessKeys =
getDataObjectFromRepo[AccessKeys](MetaDataRepository)
- private[predictionio] def getMetaDataChannels(): Channels =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[Channels]]
+ *
+ * @return An implementation of [[Channels]], depending on the runtime configuration
+ */
+ def getMetaDataChannels(): Channels =
getDataObjectFromRepo[Channels](MetaDataRepository)
- private[predictionio] def getModelDataModels(): Models =
+ /** :: DeveloperApi ::
+ * Get a data access object for [[Models]]
+ *
+ * @return An implementation of [[Models]], depending on the runtime configuration
+ */
+ def getModelDataModels(): Models =
getDataObjectFromRepo[Models](ModelDataRepository)
/** Obtains a data access object that returns [[Event]] related local data
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/config.rb
----------------------------------------------------------------------
diff --git a/docs/manual/config.rb b/docs/manual/config.rb
index 14cf91a..74628f0 100644
--- a/docs/manual/config.rb
+++ b/docs/manual/config.rb
@@ -61,7 +61,7 @@ end
# Build Settings
configure :build do
set :scheme, 'https'
- set :host, 'docs.prediction.io'
+ set :host, 'predictionio.incubator.apache.org'
set :port, 80
Slim::Engine.set_options pretty: false, sort_attrs: false
activate :asset_hash
@@ -88,7 +88,7 @@ end
# S3 Sync
activate :s3_sync do |s3_sync|
- s3_sync.bucket = 'docs.prediction.io'
+ s3_sync.bucket = 'predictionio.incubator.apache.org'
s3_sync.prefer_gzip = false
end
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/data/versions.yml
----------------------------------------------------------------------
diff --git a/docs/manual/data/versions.yml b/docs/manual/data/versions.yml
index 0768ca1..06001a7 100644
--- a/docs/manual/data/versions.yml
+++ b/docs/manual/data/versions.yml
@@ -1,6 +1,7 @@
-pio: 0.10.0-incubating
-spark: 1.5.2
-spark_download_filename: spark-1.5.1-bin-hadoop2.6
-elasticsearch_download_filename: elasticsearch-1.4.4
-hbase_basename: hbase-1.0.0
+pio: 0.11.0-incubating
+spark: 1.6.3
+spark_download_filename: spark-1.6.3-bin-hadoop2.6
+elasticsearch_download_filename: elasticsearch-1.7.6
+hbase_version: 1.2.5
+hbase_basename: hbase-1.2.5
hbase_variant: bin
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/community/contribute-code.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/community/contribute-code.html.md b/docs/manual/source/community/contribute-code.html.md
index 0d4b3fd..f1c1810 100644
--- a/docs/manual/source/community/contribute-code.html.md
+++ b/docs/manual/source/community/contribute-code.html.md
@@ -124,7 +124,7 @@ After the previous section, you should have a copy of Apache PredictionIO
Apache PredictionIO (incubating) relies on 3rd party software to perform its
tasks. To set them up, simply follow this [documentation](
-http://docs.prediction.io/install/install-sourcecode/#installing-dependencies).
+http://predictionio.incubator.apache.org/install/install-sourcecode/#installing-dependencies).
### Start Hacking
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/community/contribute-sdk.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/community/contribute-sdk.html.md b/docs/manual/source/community/contribute-sdk.html.md
index 4ee5d31..a4a6fc0 100644
--- a/docs/manual/source/community/contribute-sdk.html.md
+++ b/docs/manual/source/community/contribute-sdk.html.md
@@ -26,7 +26,7 @@ corresponding clients: `Event Client` and `Engine Client`.
The following guideline bases on the REST API provided by Apache PredictionIO
(incubating)'s Event Client which details can be found
-[here](http://docs.prediction.io/datacollection/eventapi/).
+[here](http://predictionio.incubator.apache.org/datacollection/eventapi/).
## Event Client
Because the Event Server has only 1 connection point, the `Event Client`
@@ -37,7 +37,7 @@ following rules.
(e.g. http://localhost:7070/events.json?accessKey=1234567890)
- **Request**: `POST` + JSON data. Please refer to the [Event Creation API]
-(http://docs.prediction.io/datacollection/eventapi/) for the details
+(http://predictionio.incubator.apache.org/datacollection/eventapi/) for the details
on the fields of the JSON data object.
- **Response**:
@@ -134,7 +134,7 @@ support the following 7 shorthand operations:
```
Again, please refer to the [API documentation]
-(http://docs.prediction.io/datacollection/eventapi/) for explanations
+(http://predictionio.incubator.apache.org/datacollection/eventapi/) for explanations
on the reversed events like `$set`, `$unset` or `$delete`.
INFO: The `eventTime` is optional but it is recommended that the client
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/customize/dase.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/customize/dase.html.md.erb b/docs/manual/source/customize/dase.html.md.erb
index 419abb9..89c75ea 100644
--- a/docs/manual/source/customize/dase.html.md.erb
+++ b/docs/manual/source/customize/dase.html.md.erb
@@ -27,7 +27,7 @@ DataSource reads and selects useful data from the Event Store (data store of the
## readTraining()
-You need to implment readTraining() of [PDataSource](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.PDataSource), where you can use the [PEventStore Engine API](https://docs.prediction.io/api/current/#org.apache.predictionio.data.store.PEventStore$) to read the events and create the TrainingData based on the events.
+You need to implment readTraining() of [PDataSource](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.PDataSource), where you can use the [PEventStore Engine API](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.data.store.PEventStore$) to read the events and create the TrainingData based on the events.
The following code example reads user "view" and "buy" item events, filters specific type of events for future processing and returns TrainingData accordingly.
@@ -64,9 +64,9 @@ class DataSource(val dsp: DataSourceParams)
## Using PEventStore Engine API
-Please see [Event Server Overview](https://docs.prediction.io/datacollection/) to understand [EventAPI](https://docs.prediction.io/datacollection/eventapi/) and [event modeling](https://docs.prediction.io/datacollection/eventmodel/).
+Please see [Event Server Overview](https://predictionio.incubator.apache.org/datacollection/) to understand [EventAPI](https://predictionio.incubator.apache.org/datacollection/eventapi/) and [event modeling](https://predictionio.incubator.apache.org/datacollection/eventmodel/).
-With [PEventStore Engine API](https://docs.prediction.io/api/current/#org.apache.predictionio.data.store.PEventStore$), you can easily read different events in DataSource and get the information you need.
+With [PEventStore Engine API](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.data.store.PEventStore$), you can easily read different events in DataSource and get the information you need.
For example, let's say you have events like the following:
@@ -117,7 +117,7 @@ Then following code could read these events and extract the properties field of
If you have used special events `$set/$unset/$delete` setting entity's properties, you can retrieve it with `PEventStore.aggregateProperties()`.
-Please see [event modeling](https://docs.prediction.io/datacollection/eventmodel/) to understand usage of special `$set/$unset/$delete` events.
+Please see [event modeling](https://predictionio.incubator.apache.org/datacollection/eventmodel/) to understand usage of special `$set/$unset/$delete` events.
For example, the following code show how you could retrieve properties of the "item" entities:
@@ -163,7 +163,7 @@ A few example usages of Preparator:
## prepare()
-You need to implement the `prepare()` method of [PPrepartor](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.PPreparator) to perform such tasks.
+You need to implement the `prepare()` method of [PPrepartor](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.PPreparator) to perform such tasks.
Example:
@@ -185,8 +185,8 @@ predict() is responsible for using this model to make prediction. It is called w
Apache PredictionIO (incubating) supports two types of algorithms:
-- **[P2LAlgorithm](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.P2LAlgorithm)**: trains a Model which does not contain RDD
-- **[PAlgorithm](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.PAlgorithm)**: trains a Model which contains RDD
+- **[P2LAlgorithm](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.P2LAlgorithm)**: trains a Model which does not contain RDD
+- **[PAlgorithm](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.PAlgorithm)**: trains a Model which contains RDD
## P2LAlgorithm
@@ -213,7 +213,7 @@ Example:
## using LEventStore Engine API in predict()
-You may use [LEventStore.findByEntity()](https://docs.prediction.io/api/current/#org.apache.predictionio.data.store.LEventStore$) to retrieve events of a specific entity. For example, retrieve recent events of the user specified in the query) and use these recent events to make prediction in real time.
+You may use [LEventStore.findByEntity()](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.data.store.LEventStore$) to retrieve events of a specific entity. For example, retrieve recent events of the user specified in the query) and use these recent events to make prediction in real time.
For example, the following code reads the recent 10 view events of `query.user`:
@@ -253,7 +253,7 @@ Example:
## serve()
-You need to implement the serve() method of the class [LServing](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.LServing). The serve() method processes predicted result. It is also responsible for combining multiple predicted results into one if you have more than one predictive model.
+You need to implement the serve() method of the class [LServing](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.LServing). The serve() method processes predicted result. It is also responsible for combining multiple predicted results into one if you have more than one predictive model.
Example:
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/datacollection/eventapi.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/datacollection/eventapi.html.md b/docs/manual/source/datacollection/eventapi.html.md
index ca64812..c847b48 100644
--- a/docs/manual/source/datacollection/eventapi.html.md
+++ b/docs/manual/source/datacollection/eventapi.html.md
@@ -27,14 +27,14 @@ with `EventClient`s of PredictionIO's SDKs.
INFO: All Apache PredictionIO (incubating)-compliant engines support accessing
the Event Store (i.e. the data store of Event Server) through [Apache
PredictionIO (incubating)'s Storage
-API](http://docs.prediction.io/api/current/index.html#io.prediction.data.storage.package).
+API](http://predictionio.incubator.apache.org/api/current/index.html#org.apache.predictionio.data.storage.package).
## Launching the Event Server
INFO: Before launching the Event Server, make sure that your event data store
backend is properly configured and is running. By default, Apache PredictionIO
(incubating) uses Apache HBase, and a quick configuration can be found
-[here](/install/install-linux.html#hbase). Please allow a minute (usually less
+[here](/install/install-sourcecode/#hbase). Please allow a minute (usually less
than 30 seconds) after HBase is started for its initialization to complete
before starting the Event Server.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/datacollection/eventmodel.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/datacollection/eventmodel.html.md.erb b/docs/manual/source/datacollection/eventmodel.html.md.erb
index 8a71f84..f40880c 100644
--- a/docs/manual/source/datacollection/eventmodel.html.md.erb
+++ b/docs/manual/source/datacollection/eventmodel.html.md.erb
@@ -104,7 +104,7 @@ For example, setting entity `user-1`'s properties `birthday` and `address`:
}
```
-**Note** that the properties values of the entity will be aggregated based on these special events and the eventTime. The state of the entity is different depending on the time you are looking at the data. In engine's DataSource, you can use [PEventStore.aggregateProperties() API](https://docs.prediction.io/api/current/#org.apache.predictionio.data.store.PEventStore$) to retrieve the state of entity's properties (based on time).
+**Note** that the properties values of the entity will be aggregated based on these special events and the eventTime. The state of the entity is different depending on the time you are looking at the data. In engine's DataSource, you can use [PEventStore.aggregateProperties() API](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.data.store.PEventStore$) to retrieve the state of entity's properties (based on time).
NOTE: Although it doesn't hurt to import duplicated special events for an entity (exactly same properties) into event server (it just means that the entity changes to the same state as before and new duplicated event provides no new information about the user), it could waste storage space.
@@ -240,7 +240,7 @@ NOTE: Note that you should quote the entire URL by using single or double quotes
You should see all events being created for this user-2.
-Now, let's retrieve the user-2's properties using the [PEventStore API](https://docs.prediction.io/api/current/#org.apache.predictionio.data.store.PEventStore$).
+Now, let's retrieve the user-2's properties using the [PEventStore API](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.data.store.PEventStore$).
First, start `pio-shell` by running:
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/demo/tapster.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/demo/tapster.html.md b/docs/manual/source/demo/tapster.html.md
index 97ec7f8..7345bab 100644
--- a/docs/manual/source/demo/tapster.html.md
+++ b/docs/manual/source/demo/tapster.html.md
@@ -23,7 +23,7 @@ limitations under the License.
In this demo, we will show you how to build a Tinder-style web application (named "Tapster") recommending comics to users based on their likes/dislikes of episodes interactively.
-The demo will use [Similar Product Template](https://docs.prediction.io/templates/similarproduct/quickstart/). Similar Product Template is a great choice if you want to make recommendations based on immediate user activities or for new users with limited history. It uses MLLib Alternating Least Squares (ALS) recommendation algorithm, a [Collaborative filtering](http://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering) (CF) algorithm commonly used for recommender systems. These techniques aim to fill in the missing entries of a user-item association matrix. Users and products are described by a small set of latent factors that can be used to predict missing entries. A layman's interpretation of Collaborative Filtering is "People who like this comic, also like these comics."
+The demo will use [Similar Product Template](https://predictionio.incubator.apache.org/templates/similarproduct/quickstart/). Similar Product Template is a great choice if you want to make recommendations based on immediate user activities or for new users with limited history. It uses MLLib Alternating Least Squares (ALS) recommendation algorithm, a [Collaborative filtering](http://en.wikipedia.org/wiki/Recommender_system#Collaborative_filtering) (CF) algorithm commonly used for recommender systems. These techniques aim to fill in the missing entries of a user-item association matrix. Users and products are described by a small set of latent factors that can be used to predict missing entries. A layman's interpretation of Collaborative Filtering is "People who like this comic, also like these comics."
All the code and data is on GitHub at: [github.com/PredictionIO/Demo-Tapster](https://github.com/PredictionIO/Demo-Tapster).
@@ -406,7 +406,7 @@ looking it up in the database and rendering that object as JSON.
Once the response is sent to the browser JavaScript is used to replace the existing comic and hide the loading message.
-Thats it. You\u2019re done! If Ruby is not your language of choice check out our other [SDKs](http://docs.prediction.io/sdk/) and remember you can always interact with the Event Server though it\u2019s native JSON API.
+Thats it. You\u2019re done! If Ruby is not your language of choice check out our other [SDKs](http://predictionio.incubator.apache.org/sdk/) and remember you can always interact with the Event Server though it\u2019s native JSON API.
## Links
Source code is on GitHub at: [github.com/PredictionIO/Demo-Tapster](https://github.com/PredictionIO/Demo-Tapster)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/demo/textclassification.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/demo/textclassification.html.md.erb b/docs/manual/source/demo/textclassification.html.md.erb
index cacc2f4..7beef7d 100644
--- a/docs/manual/source/demo/textclassification.html.md.erb
+++ b/docs/manual/source/demo/textclassification.html.md.erb
@@ -61,7 +61,7 @@ Here category is the model's class assignment for this new text document (i.e. t
`{category : String}`.
-This is used in the evaluation stage when estimating the performance of your predictive model (how well does the model predict categories). Please refer to the [following tutorial](https://docs.prediction.io/customize/) for a more detailed explanation of how your engine will interact with your web application, as well as an in depth-overview of DASE.
+This is used in the evaluation stage when estimating the performance of your predictive model (how well does the model predict categories). Please refer to the [following tutorial](https://predictionio.incubator.apache.org/customize/) for a more detailed explanation of how your engine will interact with your web application, as well as an in depth-overview of DASE.
## Quick Start
@@ -497,7 +497,7 @@ The last and final object implemented in this class simply creates a Map with ke
The algorithm components in this engine, `NBAlgorithm` and `LRAlgorithm`, actually follows a very general form. Firstly, a parameter class must again be initialized to feed in the corresponding Algorithm model parameters. For example, NBAlgorithm incorporates NBAlgorithmParams which holds the appropriate additive smoothing parameter lambda for the Naive Bayes model.
-The main class of interest in this component is the class that extends [P2LAlgorithm](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.P2LAlgorithm). This class must implement a method named train which will output your predictive model (as a concrete object, this will be implemented via a Scala class). It must also implement a predict method that transforms a query to an appropriate feature vector, and uses this to predict with the fitted model. The vectorization function is implemented by a PreparedData object, and the categorization (prediction) is handled by an instance of the NBModel implementation. Again, this demonstrates the facility with which different models can be incorporated into PredictionIO's DASE architecture.
+The main class of interest in this component is the class that extends [P2LAlgorithm](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.P2LAlgorithm). This class must implement a method named train which will output your predictive model (as a concrete object, this will be implemented via a Scala class). It must also implement a predict method that transforms a query to an appropriate feature vector, and uses this to predict with the fitted model. The vectorization function is implemented by a PreparedData object, and the categorization (prediction) is handled by an instance of the NBModel implementation. Again, this demonstrates the facility with which different models can be incorporated into PredictionIO's DASE architecture.
The model class itself will be discussed in the following section, however, turn your attention to the TextManipulationEngine object defined in the script `Engine.scala`. You can see here that the engine is initialized by specifying the DataSource, Preparator, and Serving classes, as well as a Map of algorithm names to Algorithm classes. This tells the engine which algorithms to run. In practice, you can have as many statistical learning models as you'd like, you simply have to implement a new algorithm component to do this. However, this general design form will persist, and the main meat of the work should be in the implementation of your model class.
@@ -607,7 +607,7 @@ To use the alternative multinomial logistic regression algorithm change your `en
## Serving: Delivering the Final Prediction
-The serving component is the final stage in the engine, and in a sense, the most important. This is the final stage in which you combine the results obtained from the different models you choose to run. The Serving class extends the [LServing](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.LServing) class which must implement a method called serve. This takes a query and an associated sequence of predicted results, which contains the predicted results from the different algorithms that are implemented in your engine, and combines the results to yield a final prediction. It is this final prediction that you will receive after sending a query.
+The serving component is the final stage in the engine, and in a sense, the most important. This is the final stage in which you combine the results obtained from the different models you choose to run. The Serving class extends the [LServing](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.LServing) class which must implement a method called serve. This takes a query and an associated sequence of predicted results, which contains the predicted results from the different algorithms that are implemented in your engine, and combines the results to yield a final prediction. It is this final prediction that you will receive after sending a query.
For example, you could choose to slightly modify the implementation to return class probabilities coming from a mixture of model estimates for class probabilities, or any other technique you could conceive for combining your results. The default engine setting has this set to yield the label from the model predicting with greater confidence.
@@ -617,7 +617,7 @@ For example, you could choose to slightly modify the implementation to return cl
A predictive model needs to be evaluated to see how it will generalize to future observations. PredictionIO uses cross-validation to perform model performance metric estimates needed to assess your particular choice of model. The script `Evaluation.scala` available with the engine template exemplifies what a usual evaluator setup will look like. First, you must define an appropriate metric. In the engine template, since the topic is text classification, the default metric implemented is category accuracy.
- Second you must define an evaluation object (i.e. extends the class [Evaluation](https://docs.prediction.io/api/current/#org.apache.predictionio.controller.Evaluation)).
+ Second you must define an evaluation object (i.e. extends the class [Evaluation](https://predictionio.incubator.apache.org/api/current/#org.apache.predictionio.controller.Evaluation)).
Here, you must specify the actual engine and metric components that are to be used for the evaluation. In the engine template, the specified engine is the TextManipulationEngine object, and metric, Accuracy. Lastly, you must specify the parameter values that you want to test in the cross validation. You see in the following block of code:
```scala
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/gallery/templates.yaml
----------------------------------------------------------------------
diff --git a/docs/manual/source/gallery/templates.yaml b/docs/manual/source/gallery/templates.yaml
index 7e1dd34..9f0ec1d 100644
--- a/docs/manual/source/gallery/templates.yaml
+++ b/docs/manual/source/gallery/templates.yaml
@@ -689,12 +689,7 @@
name: Linear Regression BFGS
repo: "https://github.com/mgcdanny/pio-linear-regression-bfgs"
description: |-
- Use for:
-
- * Modeling the relationship between a dependent variable, y, and one or more explanatory variables, denoted X.
-
- Forked from apache/incubator-predictionio-template-skeleton
-
+ Modeling the relationship between a dependent variable, y, and one or more explanatory variables, denoted X.
tags: [regression]
type: Parallel
language: Scala
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/install/index.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/index.html.md.erb b/docs/manual/source/install/index.html.md.erb
index 70879d1..902c3bc 100644
--- a/docs/manual/source/install/index.html.md.erb
+++ b/docs/manual/source/install/index.html.md.erb
@@ -19,18 +19,6 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-[//]: # (## Method 1: Quick Install)
-
-[//]: # (On Linux / Mac OS X, Apache PredictionIO (incubating) can now be installed with a single command:)
-
-[//]: # (```)
-[//]: # ($ bash -c "$(curl -s https://raw.githubusercontent.com/apache/incubator-predictionio/master/bin/install.sh)")
-[//]: # (```)
-
-[//]: # (The above script will complete the installation for you.)
-
-[//]: # (<%= partial 'shared/install/proceed_template' %>)
-
## Prerequisites
It is **very important** to meet the minimum version of the following
@@ -38,7 +26,7 @@ technologies that power Apache PredictionIO (incubating).
* Apache Hadoop 2.4.0 (optional, required only if YARN and HDFS are needed)
* Apache Spark 1.4.0 for Hadoop 2.4
-* Java SE Development Kit 7
+* Java SE Development Kit 8
and one of the following sets:
@@ -65,11 +53,7 @@ mode](http://spark.apache.org/docs/latest/spark-standalone.html).
Apache PredictionIO (incubating) runs on a Java virtual machine, so it runs on
most platforms. Choose your platform below:
-[//]: # (* [Installing Apache PredictionIO (incubating) on Linux / Mac OS X](install-linux.html))
* [Installing Apache PredictionIO (incubating) from Source Code](install-sourcecode.html)
-[//]: # (* [Installing Apache PredictionIO (incubating) with Vagrant (VirtualBox)](install-vagrant.html))
-[//]: # (* [Installing Apache PredictionIO (incubating) with Terminal.com Snap](https://www.terminal.com/snapshot/f444bfb7538dfc596485374f56167ec6f79cbc16f793f013ad120067070eb81a))
-
You may also use one of the community-contributed packages to install
Apache PredictionIO (incubating):
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/install/install-sourcecode.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/install-sourcecode.html.md.erb b/docs/manual/source/install/install-sourcecode.html.md.erb
index a795b82..7d5614f 100644
--- a/docs/manual/source/install/install-sourcecode.html.md.erb
+++ b/docs/manual/source/install/install-sourcecode.html.md.erb
@@ -32,7 +32,12 @@ data.versions.pio %>.tar.gz).
## Building
Run the following at the directory where you downloaded the source code to build
-Apache PredictionIO (incubating).
+Apache PredictionIO (incubating). By default, the build will be against
+
+* Scala 2.10.6
+* Spark 1.6.3
+* Hadoop 2.6.5
+* Elasticsearch 1.7.6
```
$ tar zxvf apache-predictionio-<%= data.versions.pio %>.tar.gz
@@ -57,6 +62,23 @@ Extract the binary distribution you have just built.
$ tar zxvf PredictionIO-<%= data.versions.pio %>.tar.gz
```
+### Building against Different Versions of Dependencies
+
+Starting from version 0.11.0, PredictionIO can be built against different
+versions of dependencies. As of writing, one could build PredictionIO against
+these different dependencies:
+
+* Scala 2.10.x, 2.11.x
+* Spark 1.6.x, 2.x
+* Hadoop 2.4.x to 2.7.x
+* Elasticsearch 1.x, 5.x
+
+As an example, if you want to build PredictionIO to support Scala 2.11.8,
+Spark 2.1.0, and Elasticsearch 5.3.0, you can do
+
+```
+$ ./make-distribution.sh -Dscala.version=2.11.8 -Dspark.version=2.1.0 -Delasticsearch.version=5.3.0
+```
## Installing Dependencies
@@ -76,7 +98,9 @@ $ mkdir PredictionIO-<%= data.versions.pio %>/vendors
### Storage Setup
-#### PostgreSQL Setup
+#### <a name="pgsql"></a>PostgreSQL Setup
+
+WARNING: You may skip this section if you are not using PostgreSQL.
<%=partial 'shared/install/postgres' %>
@@ -84,14 +108,14 @@ $ mkdir PredictionIO-<%= data.versions.pio %>/vendors
##### Elasticsearch Setup
-WARNING: You may skip this section if you are using PostgreSQL or MySQL.
+WARNING: You may skip this section if you are not using Elasticsearch.
<%= partial 'shared/install/elasticsearch' %>
##### <a name="hbase"></a>HBase Setup
-WARNING: You may skip this section if you are using PostgreSQL or MySQL.
+WARNING: You may skip this section if you are not using HBase.
<%= partial 'shared/install/hbase' %>
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/install/install-vagrant.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/install/install-vagrant.html.md.erb b/docs/manual/source/install/install-vagrant.html.md.erb
index 8cd4fd0..9e91b94 100644
--- a/docs/manual/source/install/install-vagrant.html.md.erb
+++ b/docs/manual/source/install/install-vagrant.html.md.erb
@@ -75,7 +75,7 @@ When it finishes successfully, you should see somthing like the following:
==> default: Check the status with: 'pio status'
==> default: Use: 'pio [train|deploy|...]' commands
==> default: Please report any problems to: support@prediction.io
-==> default: Documentation at: http://docs.prediction.io
+==> default: Documentation at: http://predictionio.incubator.apache.org
==> default: --------------------------------------------------------------------------------
==> default: Finish PredictionIO installation.
```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/partials/_action_call.html.slim
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/_action_call.html.slim b/docs/manual/source/partials/_action_call.html.slim
index 7554540..2b824a6 100644
--- a/docs/manual/source/partials/_action_call.html.slim
+++ b/docs/manual/source/partials/_action_call.html.slim
@@ -7,7 +7,7 @@
p
| Please take a look at the
= succeed "." do
- a href="//docs.prediction.io/" target="blank" Docs
+ a href="//predictionio.incubator.apache.org/" target="blank" Docs
| It is helpful to read through a few examples.
.col-md-4.col-xs-12
.action-square
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/partials/_footer.html.slim
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/_footer.html.slim b/docs/manual/source/partials/_footer.html.slim
index c9c9565..feb1d0c 100644
--- a/docs/manual/source/partials/_footer.html.slim
+++ b/docs/manual/source/partials/_footer.html.slim
@@ -8,8 +8,8 @@ footer
.footer-link-column-row
h4 Community
ul
- li: a href="//docs.prediction.io/install/" target="blank" Download
- li: a href="//docs.prediction.io/" target="blank" Docs
+ li: a href="//predictionio.incubator.apache.org/install/" target="blank" Download
+ li: a href="//predictionio.incubator.apache.org/" target="blank" Docs
li: a href="//github.com/apache/incubator-predictionio" target="blank" GitHub
li: a href="mailto:user-subscribe@predictionio.incubator.apache.org" target="blank" Subscribe to User Mailing List
li: a href="//stackoverflow.com/questions/tagged/predictionio" target="blank" Stackoverflow
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/partials/shared/install/_elasticsearch.html.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/shared/install/_elasticsearch.html.erb b/docs/manual/source/partials/shared/install/_elasticsearch.html.erb
index 40e16b1..5befd71 100644
--- a/docs/manual/source/partials/shared/install/_elasticsearch.html.erb
+++ b/docs/manual/source/partials/shared/install/_elasticsearch.html.erb
@@ -15,11 +15,14 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-[Elasticsearch](http://www.elasticsearch.org/) is the default metadata store for
-PredictionIO. Download and extract it.
+[Elasticsearch](https://www.elastic.co/) can be used as a storage backend for
+the meta data repository.
+
+INFO: Starting from 0.11.0, if you build PredictionIO against Elasticsearch 5+,
+you may also use it as a backend for the event data repository.
```
-$ wget https://download.elasticsearch.org/elasticsearch/elasticsearch/<%= data.versions.elasticsearch_download_filename %>.tar.gz
+$ wget https://download.elastic.co/elasticsearch/elasticsearch/<%= data.versions.elasticsearch_download_filename %>.tar.gz
$ tar zxvfC <%= data.versions.elasticsearch_download_filename %>.tar.gz PredictionIO-<%= data.versions.pio %>/vendors
```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/partials/shared/install/_hbase.html.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/shared/install/_hbase.html.erb b/docs/manual/source/partials/shared/install/_hbase.html.erb
index 78395eb..9d0cbdb 100644
--- a/docs/manual/source/partials/shared/install/_hbase.html.erb
+++ b/docs/manual/source/partials/shared/install/_hbase.html.erb
@@ -15,11 +15,15 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-[HBase](http://hbase.apache.org) is the default event data store for
-PredictionIO. Download and extract it.
+[HBase](http://hbase.apache.org) can be used as the backend of the event data
+repository.
+
+Download HBase from a [mirror](http://www.apache.org/dyn/closer.cgi/hbase/<%=
+data.versions.hbase_version %>/<%= data.versions.hbase_basename %>-<%=
+data.versions.hbase_variant %>.tar.gz). Extract HBase by following the example
+below.
```
-$ wget http://archive.apache.org/dist/hbase/<%= data.versions.hbase_basename %>/<%= data.versions.hbase_basename %>-<%= data.versions.hbase_variant %>.tar.gz
$ tar zxvfC <%= data.versions.hbase_basename %>-<%= data.versions.hbase_variant %>.tar.gz PredictionIO-<%= data.versions.pio %>/vendors
```
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/partials/shared/install/_postgres.html.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/partials/shared/install/_postgres.html.erb b/docs/manual/source/partials/shared/install/_postgres.html.erb
index 8302e5d..a2e6e99 100644
--- a/docs/manual/source/partials/shared/install/_postgres.html.erb
+++ b/docs/manual/source/partials/shared/install/_postgres.html.erb
@@ -15,25 +15,31 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
-Setting up [PostgreSQL](http://www.postgresql.org/) to work with PredictionIO.
+[PostgreSQL](https://www.postgresql.org/) can be used by PredictionIO as a
+storage backend for all 3 repositories (event data, meta data, and model data).
+This is perhaps the easiest route if you are trying PredictionIO for the first
+time.
-Make sure you have PostgreSQL installed. For Mac Users, [Homebrew](brew.sh) is recommended and can be used as
+Make sure you have PostgreSQL installed. For Mac Users,
+[Homebrew](https://brew.sh/) is recommended and can be used as
```
$ brew install postgresql
```
or on Ubuntu:
+
```
-apt-get install postgresql-9.4
+$ apt-get install postgresql
```
Now that PostgreSQL is installed use the following comands
```
$ createdb pio
-```
+```
-If you get an error of the form `could not connect to server: No such file or directory`, then you must first start the server manually,:
+If you get an error of the form `could not connect to server: No such file or
+directory`, then you must first start the server manually,:
```
$ pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start
@@ -45,4 +51,9 @@ Finally use the command:
$ psql -c "create user pio with password 'pio'"
```
-Your configuration in `pio-env.sh` is now compatible to run with PostgreSQL.
+Starting from 0.11.0, PredictionIO no longer bundles JDBC drivers. Download the
+PostgreSQL JDBC driver from the [official web
+site](https://jdbc.postgresql.org/), and put the JAR file in the `lib`
+subdirectory. By default, `conf/pio-env.sh` assumes version 42.0.0 JDBC 4.2. If
+you use a different version, modify `POSTGRES_JDBC_DRIVER` to point to the
+correct JAR.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/resources/faq.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/resources/faq.html.md b/docs/manual/source/resources/faq.html.md
index 80d98d5..a08d631 100644
--- a/docs/manual/source/resources/faq.html.md
+++ b/docs/manual/source/resources/faq.html.md
@@ -24,7 +24,7 @@ the user mailing list. You can follow the instructions [here](/support).
## Using PredictionIO
-### Q: How do I check to see if various dependencies, such as ElasticSearch and HBase, are running?
+### Q: How do I check to see if various dependencies, such as Elasticsearch and HBase, are running?
You can run `$ pio status` from the terminal and it will return the status of various components that PredictionIO depends on.
@@ -52,7 +52,7 @@ Storage Backend Connections
Your system is all ready to go.
```
-- If you see the following error message, it usually means ElasticSearch is not running properly:
+- If you see the following error message, it usually means Elasticsearch is not running properly:
```
...
@@ -66,9 +66,9 @@ Caused by: org.elasticsearch.client.transport.NoNodeAvailableException: None of
Unable to connect to all storage backend(s) successfully. Please refer to error message(s) above. Aborting.
```
-You can check if there is any ElasticSearch process by running 'jps'.
+You can check if there is any Elasticsearch process by running 'jps'.
-Please see **How to start elasticsearch** below.
+Please see **How to start Elasticsearch** below.
- If you see the following error message, it usually means HBase is not running properly:
@@ -94,22 +94,26 @@ You can check if there is any HBase-related process by running 'jps'.
Please see **How to start HBase** below.
-### Q: How to start ElasticSearch?
+### Q: How to start Elasticsearch?
-If you used the [install script](/install/install-linux/#quick-install) to install PredictionIO, the ElasticSearch is installed at `~/PredictionIO/vendors/elasticsearch-x.y.z/` where x.y.z is the
-version number (currently it's 1.4.4). To start it, run:
+If you followed the [instructions](/install/install-sourcecode/) to install
+PredictionIO, Elasticsearch would have been installed at
+`PredictionIO/vendors/elasticsearch-x.y.z/` where x.y.z is the version number.
+To start it, run:
```
$ ~/PredictionIO/vendors/elasticsearch-x.y.z/bin/elasticsearch
```
-If you didn't use install script, please go to where ElasticSearch is installed to start it.
+If you didn't use install script, please go to where Elasticsearch is installed to start it.
-INFO: It may take some time (15 seconds or so) for ElasticSearch to become ready after you start it (wait a bit before you run `pio status` again).
+INFO: It may take some time (15 seconds or so) for Elasticsearch to become ready after you start it (wait a bit before you run `pio status` again).
### Q: How to start HBase ?
-If you used the [install script](/install/install-linux/#quick-install) to install PredictionIO, the HBase is installed at `~/PredictionIO/vendors/hbase-x.y.z/` where x.y.z is the version number (currently it's 0.98.6). To start it, run:
+If you followed the [instructions](/install/install-sourcecode/) to install
+PredictionIO, the HBase is installed at `~/PredictionIO/vendors/hbase-x.y.z/`
+where x.y.z is the version number. To start it, run:
```
$ ~/PredictionIO/vendors/hbase-x.y.z/bin/start-hbase.sh
@@ -117,7 +121,8 @@ $ ~/PredictionIO/vendors/hbase-x.y.z/bin/start-hbase.sh
If you didn't use install script, please go to where HBase is installed to start it.
-INFO: It may take some time (15 seconds or so) for HBase to become ready after you start it (wait a bit before you run `pio status` again).
+INFO: It may take some time (15 seconds or so) for HBase to become ready after
+you start it (wait a bit before you run `pio status` again).
## Problem with Event Server
@@ -163,7 +168,8 @@ $ pio train -- --conf spark.akka.frameSize=1024
### Q: How to increase heap space memory for "pio deploy"?
-If you see the following error during `pio deploy`, it means there is not enough heap space memory.
+If you see the following error during `pio deploy`, it means there is not enough
+heap space memory.
```
...
@@ -172,7 +178,8 @@ If you see the following error during `pio deploy`, it means there is not enough
...
```
-To increase the heap space, specify the "-- --driver-memory " parameter in the command. For example, set the driver memory to 8G when deploy the engine:
+To increase the heap space, specify the "-- --driver-memory " parameter in the
+command. For example, set the driver memory to 8G when deploy the engine:
```
$ pio deploy -- --driver-memory 8G
@@ -206,9 +213,9 @@ drwxr-xr-x 17 yipjustin yipjustin 4096 Nov 12 00:09 ..
PredictionIO/assembly$ rm pio-assembly-0.8.1-SNAPSHOT.jar
```
-### Q: How to resolve ".......[error] (data/compile:compile) java.lang.AssertionError: assertion failed: java.lang.AutoCloseable" when ./make_distribution.sh?
+### Q: How to resolve ".......\[error\] (data/compile:compile) java.lang.AssertionError: assertion failed: java.lang.AutoCloseable" when ./make_distribution.sh?
-PredictionIO only support Java 7 or later. Please make sure you have the
+PredictionIO only support Java 8 or later. Please make sure you have the
correct Java version with the command:
```
@@ -219,9 +226,16 @@ $ javac -version
### Q: What's the difference between P- and L- prefixed classes and functions?
-PredictionIO v0.8 is built on the top of Spark, a massively scalable programming framework. A spark algorithm is different from conventional single machine algorithm in a way that spark algorithms use the [RDD](http://spark.apache.org/docs/1.0.1/programming-guide.html#resilient-distributed-datasets-rdds) abstraction as its primary data type.
+PredictionIO v0.8 is built on the top of Spark, a massively scalable programming
+framework. A spark algorithm is different from conventional single machine
+algorithm in a way that spark algorithms use the
+[RDD](http://spark.apache.org/docs/1.0.1/programming-guide.html#resilient-distributed-datasets-rdds)
+abstraction as its primary data type.
-PredictionIO framework natively support both RDD-based algorithms and traditional single-machine algorithms. For controllers prefixed by "P" (i.e. PJavaDataSource, PJavaAlgorithm), their data include RDD abstraction; For "L" controllers, they are traditional single machine algorithms.
+PredictionIO framework natively support both RDD-based algorithms and
+traditional single-machine algorithms. For controllers prefixed by "P" (i.e.
+PJavaDataSource, PJavaAlgorithm), their data include RDD abstraction; For "L"
+controllers, they are traditional single machine algorithms.
## Running HBase
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/resources/upgrade.html.md
----------------------------------------------------------------------
diff --git a/docs/manual/source/resources/upgrade.html.md b/docs/manual/source/resources/upgrade.html.md
index 2e30123..f708fb8 100644
--- a/docs/manual/source/resources/upgrade.html.md
+++ b/docs/manual/source/resources/upgrade.html.md
@@ -21,18 +21,29 @@ limitations under the License.
This page highlights major changes in each version and upgrade tools.
-# How to upgrade
+# How to Upgrade
To upgrade and use new version of PredictionIO, do the following:
-- Download and unzip the new PredictionIO binary (the download path can be found in the [Download PredictionIO section](/install/install-linux/#method-2:-manual-install))
-- Retain the setting from current PredictionIO/conf/pio-env.sh to the new PredictionIO/conf/pio-env.sh.
-- If you have added PredictionIO/bin to your `PATH` environment variable before, change it to the new PredictionIO/bin as well.
+- Download and build the new PredictionIO binary
+ [(instructions)](/install/install-sourcecode/).
+- Retain the setting from current `PredictionIO/conf/pio-env.sh` to the new
+ `PredictionIO/conf/pio-env.sh`.
+- If you have added `PredictionIO/bin` to your `PATH` environment variable before,
+ change it to the new `PredictionIO/bin` as well.
# Additional Notes for Specific Versions Upgrade
In addition, please take notes of the following for specific version upgrade.
+## Upgrade to 0.11.0
+
+Starting from 0.11.0, PredictionIO no longer bundles any JDBC drivers in the
+binary assembly. If your setup is using a JDBC backend and you run into storage
+connection errors after an upgrade, please manually install the JDBC driver. If
+you use PostgreSQL, you can find instructions
+[here](/install/install-sourcecode#pgsql).
+
## Upgrade to 0.9.2
The Spark dependency has been upgraded to version 1.3.0. All engines must be
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/sdk/java.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/sdk/java.html.md.erb b/docs/manual/source/sdk/java.html.md.erb
index 3923635..7b1cc28 100644
--- a/docs/manual/source/sdk/java.html.md.erb
+++ b/docs/manual/source/sdk/java.html.md.erb
@@ -21,7 +21,7 @@ limitations under the License.
## API Docs
-[View API documentation](http://docs.prediction.io/java/api/) for version <%= data.versions.pio %>.
+[View API documentation](http://predictionio.incubator.apache.org/java/api/) for version <%= data.versions.pio %>.
## Central Repository
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/sdk/php.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/sdk/php.html.md.erb b/docs/manual/source/sdk/php.html.md.erb
index 4a9930c..b51a4c3 100644
--- a/docs/manual/source/sdk/php.html.md.erb
+++ b/docs/manual/source/sdk/php.html.md.erb
@@ -21,7 +21,7 @@ limitations under the License.
## API Docs
-[View API documentation](http://docs.prediction.io/php/api/) for version <%= data.versions.pio %>.
+[View API documentation](http://predictionio.incubator.apache.org/php/api/) for version <%= data.versions.pio %>.
## Packagist
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/sdk/ruby.html.md.erb
----------------------------------------------------------------------
diff --git a/docs/manual/source/sdk/ruby.html.md.erb b/docs/manual/source/sdk/ruby.html.md.erb
index b6548f7..ef08872 100644
--- a/docs/manual/source/sdk/ruby.html.md.erb
+++ b/docs/manual/source/sdk/ruby.html.md.erb
@@ -21,7 +21,7 @@ limitations under the License.
## API Docs
-[View API documentation](http://docs.prediction.io/ruby/api/PredictionIO.html) for version <%= data.versions.pio %>.
+[View API documentation](http://predictionio.incubator.apache.org/ruby/api/PredictionIO.html) for version <%= data.versions.pio %>.
## RubyGems
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/docs/manual/source/tryit/index.html.slim
----------------------------------------------------------------------
diff --git a/docs/manual/source/tryit/index.html.slim b/docs/manual/source/tryit/index.html.slim
index ee50c30..7918812 100644
--- a/docs/manual/source/tryit/index.html.slim
+++ b/docs/manual/source/tryit/index.html.slim
@@ -53,7 +53,7 @@ title: Try PredictionIO
li
h2 Next Steps
- p #{link_to 'Install', 'http://docs.prediction.io/install/'} PredictionIO on your own computer!
+ p #{link_to 'Install', 'http://predictionio.incubator.apache.org/install/'} PredictionIO on your own computer!
h3 Support
p Get support on our #{link_to 'Google Group', 'https://groups.google.com/forum/#!forum/predictionio-user'} or on Twitter #{link_to '@PredictionIO', 'https://twitter.com/predictionio/'}
.jcarousel-controls
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/tests/docker-files/env-conf/pio-env.sh
----------------------------------------------------------------------
diff --git a/tests/docker-files/env-conf/pio-env.sh b/tests/docker-files/env-conf/pio-env.sh
index 16f1fef..72a8f12 100644
--- a/tests/docker-files/env-conf/pio-env.sh
+++ b/tests/docker-files/env-conf/pio-env.sh
@@ -52,7 +52,7 @@ PIO_FS_TMPDIR=$PIO_FS_BASEDIR/tmp
# storage facilities. Default values are shown below.
#
# For more information on storage configuration please refer to
-# https://docs.prediction.io/system/anotherdatastore/
+# https://predictionio.incubator.apache.org/system/anotherdatastore/
# Storage Repositories
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/tests/pio_tests/engines/recommendation-engine/README.md
----------------------------------------------------------------------
diff --git a/tests/pio_tests/engines/recommendation-engine/README.md b/tests/pio_tests/engines/recommendation-engine/README.md
index 298321b..69bd6d9 100644
--- a/tests/pio_tests/engines/recommendation-engine/README.md
+++ b/tests/pio_tests/engines/recommendation-engine/README.md
@@ -19,7 +19,7 @@ limitations under the License.
## Documentation
-Please refer to http://docs.prediction.io/templates/recommendation/quickstart/
+Please refer to http://predictionio.incubator.apache.org/templates/recommendation/quickstart/
## Versions
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/tests/pio_tests/scenarios/eventserver_test.py
----------------------------------------------------------------------
diff --git a/tests/pio_tests/scenarios/eventserver_test.py b/tests/pio_tests/scenarios/eventserver_test.py
index 6f29876..f902f74 100644
--- a/tests/pio_tests/scenarios/eventserver_test.py
+++ b/tests/pio_tests/scenarios/eventserver_test.py
@@ -28,8 +28,8 @@ from pio_tests.integration import BaseTestCase, AppContext
class EventserverTest(BaseTestCase):
""" Integration test for PredictionIO Eventserver API
Refer to below for further information:
- http://docs.prediction.io/datacollection/eventmodel/
- http://docs.prediction.io/datacollection/eventapi/
+ http://predictionio.incubator.apache.org/datacollection/eventmodel/
+ http://predictionio.incubator.apache.org/datacollection/eventapi/
"""
# Helper methods
def eventserver_url(self, path=None):
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/tests/pio_tests/utils.py
----------------------------------------------------------------------
diff --git a/tests/pio_tests/utils.py b/tests/pio_tests/utils.py
index 05c8d1c..b5e5fac 100644
--- a/tests/pio_tests/utils.py
+++ b/tests/pio_tests/utils.py
@@ -119,7 +119,7 @@ def send_events_batch(events, test_context, access_key, channel=None):
channel (str): custom channel for storing event
Returns: `requests.Response`
Requires: Events length must not exceed length of 50
- http://docs.prediction.io/datacollection/eventmodel/#3.-batch-events-to-the-eventserver
+ http://predictionio.incubator.apache.org/datacollection/eventmodel/#3.-batch-events-to-the-eventserver
"""
url = 'http://{}:{}/batch/events.json'.format(
test_context.es_ip, test_context.es_port)
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/bea82a6c/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index 47c9f3b..84242ea 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -158,10 +158,13 @@ object Management extends EitherLogging {
case e: Throwable =>
val errStr = s"""Unable to connect to all storage backends successfully.
|The following shows the error message from the storage backend.
- |${e.getMessage} (${e.getClass.getName})", e)
+ |
+ |${e.getMessage} (${e.getClass.getName})
+ |
|Dumping configuration of initialized storage backend sources.
- |"Please make sure they are correct.
- |"""
+ |Please make sure they are correct.
+ |
+ |""".stripMargin
val sources = storage.Storage.config.get("sources") map { src =>
src map { case (s, p) =>
s"Source Name: $s; Type: ${p.getOrElse("type", "(error)")}; " +
[50/50] [abbrv] incubator-predictionio git commit: Add missing
license header
Posted by do...@apache.org.
Add missing license header
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/e34a853d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/e34a853d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/e34a853d
Branch: refs/heads/livedoc
Commit: e34a853d0e89baed09b3d3b0c25b244162a3bdea
Parents: 991b88f
Author: Donald Szeto <do...@apache.org>
Authored: Sun Apr 9 16:39:57 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Apr 9 16:39:57 2017 -0700
----------------------------------------------------------------------
PMC.md | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/e34a853d/PMC.md
----------------------------------------------------------------------
diff --git a/PMC.md b/PMC.md
index a8be0b6..1a6eb06 100644
--- a/PMC.md
+++ b/PMC.md
@@ -1,3 +1,20 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
# Project Management Committee Documentation
## Release Procedure
@@ -19,8 +36,6 @@ for staging release candidates for voting.
1. `git archive --format tar v0.11.0-incubating-rc1 >
../apache-predictionio-0.11.0-incubating-rc1.tar`
2. `cd ..; gzip apache-predictionio-0.11.0-incubating-rc1.tar`
- 3. Go to the parent directory and do `tar zcvf
- apache-predictionio-0.11.0-incubating-rc1.tar.gz predictionio`
6. Generate detached signature for the release candidate.
(http://apache.org/dev/release-signing.html#openpgp-ascii-detach-sig)
1. `gpg --armor --output apache-predictionio-0.11.0-incubating-rc1.tar.gz.asc
[15/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
[PIO-49] Add support for Elasticsearch 5
* Add support for Elasticsearch 5 over REST API
* Refactor storage implementations to submodules
* Build storage implementation as separate assemblies
Closes #352
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d78b3cbe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d78b3cbe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d78b3cbe
Branch: refs/heads/livedoc
Commit: d78b3cbe912cf57e2f0278e407a0d6432bd12849
Parents: 8fd59fd
Author: Shinsuke Sugaya <sh...@yahoo.co.jp>
Authored: Tue Mar 7 20:33:30 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Tue Mar 7 20:33:30 2017 -0800
----------------------------------------------------------------------
bin/compute-classpath.sh | 2 +-
build.sbt | 28 +-
conf/pio-env.sh.template | 10 +-
core/build.sbt | 1 -
data/build.sbt | 14 -
.../storage/elasticsearch/ESAccessKeys.scala | 119 ------
.../data/storage/elasticsearch/ESApps.scala | 130 ------
.../data/storage/elasticsearch/ESChannels.scala | 117 ------
.../elasticsearch/ESEngineInstances.scala | 158 -------
.../elasticsearch/ESEvaluationInstances.scala | 136 ------
.../storage/elasticsearch/ESSequences.scala | 64 ---
.../data/storage/elasticsearch/ESUtils.scala | 48 ---
.../storage/elasticsearch/StorageClient.scala | 50 ---
.../data/storage/elasticsearch/package.scala | 25 --
.../data/storage/hbase/HBEventsUtil.scala | 415 -------------------
.../data/storage/hbase/HBLEvents.scala | 195 ---------
.../data/storage/hbase/HBPEvents.scala | 131 ------
.../data/storage/hbase/PIOHBaseUtil.scala | 32 --
.../data/storage/hbase/StorageClient.scala | 86 ----
.../data/storage/hbase/package.scala | 25 --
.../data/storage/hbase/upgrade/HB_0_8_0.scala | 193 ---------
.../data/storage/hbase/upgrade/Upgrade.scala | 75 ----
.../storage/hbase/upgrade/Upgrade_0_8_3.scala | 224 ----------
.../data/storage/hdfs/HDFSModels.scala | 63 ---
.../data/storage/hdfs/StorageClient.scala | 36 --
.../data/storage/hdfs/package.scala | 25 --
.../data/storage/jdbc/JDBCAccessKeys.scala | 87 ----
.../data/storage/jdbc/JDBCApps.scala | 89 ----
.../data/storage/jdbc/JDBCChannels.scala | 69 ---
.../data/storage/jdbc/JDBCEngineInstances.scala | 197 ---------
.../storage/jdbc/JDBCEvaluationInstances.scala | 165 --------
.../data/storage/jdbc/JDBCLEvents.scala | 244 -----------
.../data/storage/jdbc/JDBCModels.scala | 55 ---
.../data/storage/jdbc/JDBCPEvents.scala | 188 ---------
.../data/storage/jdbc/JDBCUtils.scala | 106 -----
.../data/storage/jdbc/StorageClient.scala | 53 ---
.../data/storage/jdbc/package.scala | 26 --
.../data/storage/localfs/LocalFSModels.scala | 62 ---
.../data/storage/localfs/StorageClient.scala | 46 --
.../data/storage/localfs/package.scala | 25 --
.../predictionio/data/view/PBatchView.scala | 212 ----------
make-distribution.sh | 6 +-
storage/elasticsearch/.gitignore | 1 +
storage/elasticsearch/build.sbt | 57 +++
.../storage/elasticsearch/ESAccessKeys.scala | 178 ++++++++
.../data/storage/elasticsearch/ESApps.scala | 194 +++++++++
.../data/storage/elasticsearch/ESChannels.scala | 165 ++++++++
.../elasticsearch/ESEngineInstances.scala | 248 +++++++++++
.../elasticsearch/ESEvaluationInstances.scala | 194 +++++++++
.../storage/elasticsearch/ESEventsUtil.scala | 123 ++++++
.../data/storage/elasticsearch/ESLEvents.scala | 291 +++++++++++++
.../data/storage/elasticsearch/ESPEvents.scala | 144 +++++++
.../storage/elasticsearch/ESSequences.scala | 79 ++++
.../data/storage/elasticsearch/ESUtils.scala | 184 ++++++++
.../storage/elasticsearch/StorageClient.scala | 44 ++
.../data/storage/elasticsearch/package.scala | 25 ++
.../src/test/resources/application.conf | 28 ++
storage/elasticsearch1/.gitignore | 1 +
storage/elasticsearch1/build.sbt | 47 +++
.../storage/elasticsearch/ESAccessKeys.scala | 119 ++++++
.../data/storage/elasticsearch/ESApps.scala | 130 ++++++
.../data/storage/elasticsearch/ESChannels.scala | 117 ++++++
.../elasticsearch/ESEngineInstances.scala | 158 +++++++
.../elasticsearch/ESEvaluationInstances.scala | 136 ++++++
.../storage/elasticsearch/ESSequences.scala | 64 +++
.../data/storage/elasticsearch/ESUtils.scala | 48 +++
.../storage/elasticsearch/StorageClient.scala | 50 +++
.../data/storage/elasticsearch/package.scala | 25 ++
.../src/test/resources/application.conf | 28 ++
storage/hbase/.gitignore | 1 +
storage/hbase/build.sbt | 56 +++
.../data/storage/hbase/HBEventsUtil.scala | 415 +++++++++++++++++++
.../data/storage/hbase/HBLEvents.scala | 195 +++++++++
.../data/storage/hbase/HBPEvents.scala | 131 ++++++
.../data/storage/hbase/PIOHBaseUtil.scala | 32 ++
.../data/storage/hbase/StorageClient.scala | 86 ++++
.../data/storage/hbase/package.scala | 25 ++
.../data/storage/hbase/upgrade/HB_0_8_0.scala | 193 +++++++++
.../data/storage/hbase/upgrade/Upgrade.scala | 75 ++++
.../storage/hbase/upgrade/Upgrade_0_8_3.scala | 224 ++++++++++
.../predictionio/data/view/PBatchView.scala | 212 ++++++++++
.../hbase/src/test/resources/application.conf | 28 ++
storage/hdfs/.gitignore | 1 +
storage/hdfs/build.sbt | 44 ++
.../data/storage/hdfs/HDFSModels.scala | 63 +++
.../data/storage/hdfs/StorageClient.scala | 36 ++
.../data/storage/hdfs/package.scala | 25 ++
.../hdfs/src/test/resources/application.conf | 28 ++
storage/jdbc/.gitignore | 1 +
storage/jdbc/build.sbt | 47 +++
.../data/storage/jdbc/JDBCAccessKeys.scala | 87 ++++
.../data/storage/jdbc/JDBCApps.scala | 89 ++++
.../data/storage/jdbc/JDBCChannels.scala | 69 +++
.../data/storage/jdbc/JDBCEngineInstances.scala | 197 +++++++++
.../storage/jdbc/JDBCEvaluationInstances.scala | 165 ++++++++
.../data/storage/jdbc/JDBCLEvents.scala | 244 +++++++++++
.../data/storage/jdbc/JDBCModels.scala | 55 +++
.../data/storage/jdbc/JDBCPEvents.scala | 188 +++++++++
.../data/storage/jdbc/JDBCUtils.scala | 106 +++++
.../data/storage/jdbc/StorageClient.scala | 53 +++
.../data/storage/jdbc/package.scala | 26 ++
.../jdbc/src/test/resources/application.conf | 28 ++
storage/localfs/.gitignore | 1 +
storage/localfs/build.sbt | 44 ++
.../data/storage/localfs/LocalFSModels.scala | 62 +++
.../data/storage/localfs/StorageClient.scala | 46 ++
.../data/storage/localfs/package.scala | 25 ++
.../localfs/src/test/resources/application.conf | 28 ++
tests/Dockerfile | 10 +-
tests/build-docker.sh | 8 +-
tests/docker-compose.yml | 2 +-
tests/docker-files/env-conf/pio-env.sh | 3 +-
tests/pio_tests/scenarios/eventserver_test.py | 5 +-
tests/run_docker.sh | 4 +-
.../org/apache/predictionio/tools/Common.scala | 7 +
.../org/apache/predictionio/tools/Runner.scala | 3 +-
.../predictionio/tools/commands/Engine.scala | 4 +-
117 files changed, 6382 insertions(+), 4005 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 3bf6814..69cbb25 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -27,7 +27,7 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
# Build up classpath
CLASSPATH="${FWDIR}/conf"
-CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*"
+CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*"
ASSEMBLY_DIR="${FWDIR}/assembly"
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index eeb3724..98444b9 100644
--- a/build.sbt
+++ b/build.sbt
@@ -34,11 +34,9 @@ fork in (ThisBuild, run) := true
javacOptions in (ThisBuild, compile) ++= Seq("-source", "1.7", "-target", "1.7",
"-Xlint:deprecation", "-Xlint:unchecked")
-elasticsearchVersion in ThisBuild := "1.4.4"
-
json4sVersion in ThisBuild := "3.2.10"
-sparkVersion in ThisBuild := "1.4.0"
+sparkVersion in ThisBuild := "1.6.3"
val pioBuildInfoSettings = buildInfoSettings ++ Seq(
sourceGenerators in Compile <+= buildInfo,
@@ -65,6 +63,30 @@ val data = (project in file("data")).
settings(commonSettings: _*).
settings(genjavadocSettings: _*)
+val dataElasticsearch1 = (project in file("storage/elasticsearch1")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
+val dataElasticsearch = (project in file("storage/elasticsearch")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
+val dataHbase = (project in file("storage/hbase")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
+val dataHdfs = (project in file("storage/hdfs")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
+val dataJdbc = (project in file("storage/jdbc")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
+val dataLocalfs = (project in file("storage/localfs")).
+ settings(commonSettings: _*).
+ settings(genjavadocSettings: _*)
+
val core = (project in file("core")).
dependsOn(data).
settings(commonSettings: _*).
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/conf/pio-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/pio-env.sh.template b/conf/pio-env.sh.template
index a06cd8e..0d76102 100644
--- a/conf/pio-env.sh.template
+++ b/conf/pio-env.sh.template
@@ -24,7 +24,7 @@
# you need to change these to fit your site.
# SPARK_HOME: Apache Spark is a hard dependency and must be configured.
-SPARK_HOME=$PIO_HOME/vendors/spark-1.5.1-bin-hadoop2.6
+SPARK_HOME=$PIO_HOME/vendors/spark-1.6.3-bin-hadoop2.6
POSTGRES_JDBC_DRIVER=$PIO_HOME/lib/postgresql-9.4-1204.jdbc41.jar
MYSQL_JDBC_DRIVER=$PIO_HOME/lib/mysql-connector-java-5.1.37.jar
@@ -85,10 +85,16 @@ PIO_STORAGE_SOURCES_PGSQL_PASSWORD=pio
# Elasticsearch Example
# PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9200
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_SCHEMES=http
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-5.2.1
+# Elasticsearch 1.x Example
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_TYPE=elasticsearch
# PIO_STORAGE_SOURCES_ELASTICSEARCH_CLUSTERNAME=<elasticsearch_cluster_name>
# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOSTS=localhost
# PIO_STORAGE_SOURCES_ELASTICSEARCH_PORTS=9300
-# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-1.4.4
+# PIO_STORAGE_SOURCES_ELASTICSEARCH_HOME=$PIO_HOME/vendors/elasticsearch-1.7.6
# Local File System Example
# PIO_STORAGE_SOURCES_LOCALFS_TYPE=localfs
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
index 637d4ea..bfb8bf3 100644
--- a/core/build.sbt
+++ b/core/build.sbt
@@ -32,7 +32,6 @@ libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
- "org.elasticsearch" % "elasticsearch" % elasticsearchVersion.value,
"org.json4s" %% "json4s-native" % json4sVersion.value,
"org.json4s" %% "json4s-ext" % json4sVersion.value,
"org.scalaj" %% "scalaj-http" % "1.1.6",
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/build.sbt
----------------------------------------------------------------------
diff --git a/data/build.sbt b/data/build.sbt
index 4526c39..f5e95b5 100644
--- a/data/build.sbt
+++ b/data/build.sbt
@@ -26,29 +26,15 @@ libraryDependencies ++= Seq(
"mysql" % "mysql-connector-java" % "5.1.37" % "optional",
"org.apache.hadoop" % "hadoop-common" % "2.6.2"
exclude("javax.servlet", "servlet-api"),
- "org.apache.hbase" % "hbase-common" % "0.98.5-hadoop2",
- "org.apache.hbase" % "hbase-client" % "0.98.5-hadoop2"
- exclude("org.apache.zookeeper", "zookeeper"),
- // added for Parallel storage interface
- "org.apache.hbase" % "hbase-server" % "0.98.5-hadoop2"
- exclude("org.apache.hbase", "hbase-client")
- exclude("org.apache.zookeeper", "zookeeper")
- exclude("javax.servlet", "servlet-api")
- exclude("org.mortbay.jetty", "servlet-api-2.5")
- exclude("org.mortbay.jetty", "jsp-api-2.1")
- exclude("org.mortbay.jetty", "jsp-2.1"),
"org.apache.zookeeper" % "zookeeper" % "3.4.7"
exclude("org.slf4j", "slf4j-api")
exclude("org.slf4j", "slf4j-log4j12"),
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
"org.clapper" %% "grizzled-slf4j" % "1.0.2",
- "org.elasticsearch" % "elasticsearch" % elasticsearchVersion.value,
"org.json4s" %% "json4s-native" % json4sVersion.value,
"org.json4s" %% "json4s-ext" % json4sVersion.value,
- "org.postgresql" % "postgresql" % "9.4-1204-jdbc41",
"org.scalatest" %% "scalatest" % "2.1.7" % "test",
- "org.scalikejdbc" %% "scalikejdbc" % "2.3.5",
"org.slf4j" % "slf4j-log4j12" % "1.7.18",
"org.spark-project.akka" %% "akka-actor" % "2.3.4-spark",
"org.specs2" %% "specs2" % "2.3.13" % "test")
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
deleted file mode 100644
index 077168a..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.AccessKey
-import org.apache.predictionio.data.storage.AccessKeys
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.elasticsearch.index.query.FilterBuilders._
-import org.json4s.JsonDSL._
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-
-import scala.util.Random
-
-/** Elasticsearch implementation of AccessKeys. */
-class ESAccessKeys(client: Client, config: StorageClientConfig, index: String)
- extends AccessKeys with Logging {
- implicit val formats = DefaultFormats.lossless
- private val estype = "accesskeys"
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val json =
- (estype ->
- ("properties" ->
- ("key" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("events" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(json))).get
- }
-
- def insert(accessKey: AccessKey): Option[String] = {
- val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
- update(accessKey.copy(key = key))
- Some(key)
- }
-
- def get(key: String): Option[AccessKey] = {
- try {
- val response = client.prepareGet(
- index,
- estype,
- key).get()
- Some(read[AccessKey](response.getSourceAsString))
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- case e: NullPointerException => None
- }
- }
-
- def getAll(): Seq[AccessKey] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype)
- ESUtils.getAll[AccessKey](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq[AccessKey]()
- }
- }
-
- def getByAppid(appid: Int): Seq[AccessKey] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype).
- setPostFilter(termFilter("appid", appid))
- ESUtils.getAll[AccessKey](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq[AccessKey]()
- }
- }
-
- def update(accessKey: AccessKey): Unit = {
- try {
- client.prepareIndex(index, estype, accessKey.key).setSource(write(accessKey)).get()
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- }
- }
-
- def delete(key: String): Unit = {
- try {
- client.prepareDelete(index, estype, key).get
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
deleted file mode 100644
index 3781a4b..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.App
-import org.apache.predictionio.data.storage.Apps
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.elasticsearch.index.query.FilterBuilders._
-import org.json4s.JsonDSL._
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-
-/** Elasticsearch implementation of Items. */
-class ESApps(client: Client, config: StorageClientConfig, index: String)
- extends Apps with Logging {
- implicit val formats = DefaultFormats.lossless
- private val estype = "apps"
- private val seq = new ESSequences(client, config, index)
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val json =
- (estype ->
- ("properties" ->
- ("name" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(json))).get
- }
-
- def insert(app: App): Option[Int] = {
- val id =
- if (app.id == 0) {
- var roll = seq.genNext("apps")
- while (!get(roll).isEmpty) roll = seq.genNext("apps")
- roll
- }
- else app.id
- val realapp = app.copy(id = id)
- update(realapp)
- Some(id)
- }
-
- def get(id: Int): Option[App] = {
- try {
- val response = client.prepareGet(
- index,
- estype,
- id.toString).get()
- Some(read[App](response.getSourceAsString))
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- case e: NullPointerException => None
- }
- }
-
- def getByName(name: String): Option[App] = {
- try {
- val response = client.prepareSearch(index).setTypes(estype).
- setPostFilter(termFilter("name", name)).get
- val hits = response.getHits().hits()
- if (hits.size > 0) {
- Some(read[App](hits.head.getSourceAsString))
- } else {
- None
- }
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- }
- }
-
- def getAll(): Seq[App] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype)
- ESUtils.getAll[App](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq[App]()
- }
- }
-
- def update(app: App): Unit = {
- try {
- val response = client.prepareIndex(index, estype, app.id.toString).
- setSource(write(app)).get()
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- }
- }
-
- def delete(id: Int): Unit = {
- try {
- client.prepareDelete(index, estype, id.toString).get
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
deleted file mode 100644
index 52697fd..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Channel
-import org.apache.predictionio.data.storage.Channels
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.elasticsearch.index.query.FilterBuilders.termFilter
-import org.json4s.DefaultFormats
-import org.json4s.JsonDSL._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-
-class ESChannels(client: Client, config: StorageClientConfig, index: String)
- extends Channels with Logging {
-
- implicit val formats = DefaultFormats.lossless
- private val estype = "channels"
- private val seq = new ESSequences(client, config, index)
- private val seqName = "channels"
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val json =
- (estype ->
- ("properties" ->
- ("name" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(json))).get
- }
-
- def insert(channel: Channel): Option[Int] = {
- val id =
- if (channel.id == 0) {
- var roll = seq.genNext(seqName)
- while (!get(roll).isEmpty) roll = seq.genNext(seqName)
- roll
- } else channel.id
-
- val realChannel = channel.copy(id = id)
- if (update(realChannel)) Some(id) else None
- }
-
- def get(id: Int): Option[Channel] = {
- try {
- val response = client.prepareGet(
- index,
- estype,
- id.toString).get()
- Some(read[Channel](response.getSourceAsString))
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- case e: NullPointerException => None
- }
- }
-
- def getByAppid(appid: Int): Seq[Channel] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype).
- setPostFilter(termFilter("appid", appid))
- ESUtils.getAll[Channel](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq[Channel]()
- }
- }
-
- def update(channel: Channel): Boolean = {
- try {
- val response = client.prepareIndex(index, estype, channel.id.toString).
- setSource(write(channel)).get()
- true
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- false
- }
- }
-
- def delete(id: Int): Unit = {
- try {
- client.prepareDelete(index, estype, id.toString).get
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
deleted file mode 100644
index 21690bf..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.EngineInstance
-import org.apache.predictionio.data.storage.EngineInstanceSerializer
-import org.apache.predictionio.data.storage.EngineInstances
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.elasticsearch.index.query.FilterBuilders._
-import org.elasticsearch.search.sort.SortOrder
-import org.json4s.JsonDSL._
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-
-class ESEngineInstances(client: Client, config: StorageClientConfig, index: String)
- extends EngineInstances with Logging {
- implicit val formats = DefaultFormats + new EngineInstanceSerializer
- private val estype = "engine_instances"
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val json =
- (estype ->
- ("properties" ->
- ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("startTime" -> ("type" -> "date")) ~
- ("endTime" -> ("type" -> "date")) ~
- ("engineId" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("engineVersion" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("engineVariant" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("engineFactory" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("batch" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("dataSourceParams" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("preparatorParams" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("algorithmsParams" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("servingParams" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed"))))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(json))).get
- }
-
- def insert(i: EngineInstance): String = {
- try {
- val response = client.prepareIndex(index, estype).
- setSource(write(i)).get
- response.getId
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- ""
- }
- }
-
- def get(id: String): Option[EngineInstance] = {
- try {
- val response = client.prepareGet(index, estype, id).get
- if (response.isExists) {
- Some(read[EngineInstance](response.getSourceAsString))
- } else {
- None
- }
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- }
- }
-
- def getAll(): Seq[EngineInstance] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype)
- ESUtils.getAll[EngineInstance](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq()
- }
- }
-
- def getCompleted(
- engineId: String,
- engineVersion: String,
- engineVariant: String): Seq[EngineInstance] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype).setPostFilter(
- andFilter(
- termFilter("status", "COMPLETED"),
- termFilter("engineId", engineId),
- termFilter("engineVersion", engineVersion),
- termFilter("engineVariant", engineVariant))).
- addSort("startTime", SortOrder.DESC)
- ESUtils.getAll[EngineInstance](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq()
- }
- }
-
- def getLatestCompleted(
- engineId: String,
- engineVersion: String,
- engineVariant: String): Option[EngineInstance] =
- getCompleted(
- engineId,
- engineVersion,
- engineVariant).headOption
-
- def update(i: EngineInstance): Unit = {
- try {
- client.prepareUpdate(index, estype, i.id).setDoc(write(i)).get
- } catch {
- case e: ElasticsearchException => error(e.getMessage)
- }
- }
-
- def delete(id: String): Unit = {
- try {
- val response = client.prepareDelete(index, estype, id).get
- } catch {
- case e: ElasticsearchException => error(e.getMessage)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
deleted file mode 100644
index 85bf820..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.EvaluationInstance
-import org.apache.predictionio.data.storage.EvaluationInstanceSerializer
-import org.apache.predictionio.data.storage.EvaluationInstances
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.elasticsearch.index.query.FilterBuilders._
-import org.elasticsearch.search.sort.SortOrder
-import org.json4s.JsonDSL._
-import org.json4s._
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.read
-import org.json4s.native.Serialization.write
-
-class ESEvaluationInstances(client: Client, config: StorageClientConfig, index: String)
- extends EvaluationInstances with Logging {
- implicit val formats = DefaultFormats + new EvaluationInstanceSerializer
- private val estype = "evaluation_instances"
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val json =
- (estype ->
- ("properties" ->
- ("status" -> ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("startTime" -> ("type" -> "date")) ~
- ("endTime" -> ("type" -> "date")) ~
- ("evaluationClass" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("engineParamsGeneratorClass" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("batch" ->
- ("type" -> "string") ~ ("index" -> "not_analyzed")) ~
- ("evaluatorResults" ->
- ("type" -> "string") ~ ("index" -> "no")) ~
- ("evaluatorResultsHTML" ->
- ("type" -> "string") ~ ("index" -> "no")) ~
- ("evaluatorResultsJSON" ->
- ("type" -> "string") ~ ("index" -> "no"))))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(json))).get
- }
-
- def insert(i: EvaluationInstance): String = {
- try {
- val response = client.prepareIndex(index, estype).
- setSource(write(i)).get
- response.getId
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- ""
- }
- }
-
- def get(id: String): Option[EvaluationInstance] = {
- try {
- val response = client.prepareGet(index, estype, id).get
- if (response.isExists) {
- Some(read[EvaluationInstance](response.getSourceAsString))
- } else {
- None
- }
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- None
- }
- }
-
- def getAll(): Seq[EvaluationInstance] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype)
- ESUtils.getAll[EvaluationInstance](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq()
- }
- }
-
- def getCompleted(): Seq[EvaluationInstance] = {
- try {
- val builder = client.prepareSearch(index).setTypes(estype).setPostFilter(
- termFilter("status", "EVALCOMPLETED")).
- addSort("startTime", SortOrder.DESC)
- ESUtils.getAll[EvaluationInstance](client, builder)
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- Seq()
- }
- }
-
- def update(i: EvaluationInstance): Unit = {
- try {
- client.prepareUpdate(index, estype, i.id).setDoc(write(i)).get
- } catch {
- case e: ElasticsearchException => error(e.getMessage)
- }
- }
-
- def delete(id: String): Unit = {
- try {
- client.prepareDelete(index, estype, id).get
- } catch {
- case e: ElasticsearchException => error(e.getMessage)
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
deleted file mode 100644
index 5c9e170..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.elasticsearch.ElasticsearchException
-import org.elasticsearch.client.Client
-import org.json4s.JsonDSL._
-import org.json4s._
-import org.json4s.native.JsonMethods._
-
-class ESSequences(client: Client, config: StorageClientConfig, index: String) extends Logging {
- implicit val formats = DefaultFormats
- private val estype = "sequences"
-
- val indices = client.admin.indices
- val indexExistResponse = indices.prepareExists(index).get
- if (!indexExistResponse.isExists) {
- // val settingsJson =
- // ("number_of_shards" -> 1) ~
- // ("auto_expand_replicas" -> "0-all")
- indices.prepareCreate(index).get
- }
- val typeExistResponse = indices.prepareTypesExists(index).setTypes(estype).get
- if (!typeExistResponse.isExists) {
- val mappingJson =
- (estype ->
- ("_source" -> ("enabled" -> 0)) ~
- ("_all" -> ("enabled" -> 0)) ~
- ("_type" -> ("index" -> "no")) ~
- ("enabled" -> 0))
- indices.preparePutMapping(index).setType(estype).
- setSource(compact(render(mappingJson))).get
- }
-
- def genNext(name: String): Int = {
- try {
- val response = client.prepareIndex(index, estype, name).
- setSource(compact(render("n" -> name))).get
- response.getVersion().toInt
- } catch {
- case e: ElasticsearchException =>
- error(e.getMessage)
- 0
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
deleted file mode 100644
index f5c99bf..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import org.elasticsearch.action.search.SearchRequestBuilder
-import org.elasticsearch.client.Client
-import org.elasticsearch.common.unit.TimeValue
-import org.json4s.Formats
-import org.json4s.native.Serialization.read
-
-import scala.collection.mutable.ArrayBuffer
-
-object ESUtils {
- val scrollLife = new TimeValue(60000)
-
- def getAll[T : Manifest](
- client: Client,
- builder: SearchRequestBuilder)(
- implicit formats: Formats): Seq[T] = {
- val results = ArrayBuffer[T]()
- var response = builder.setScroll(scrollLife).get
- var hits = response.getHits().hits()
- results ++= hits.map(h => read[T](h.getSourceAsString))
- while (hits.size > 0) {
- response = client.prepareSearchScroll(response.getScrollId).
- setScroll(scrollLife).get
- hits = response.getHits().hits()
- results ++= hits.map(h => read[T](h.getSourceAsString))
- }
- results
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
deleted file mode 100644
index 75ac2b0..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/StorageClient.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.elasticsearch
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.BaseStorageClient
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.StorageClientException
-import org.elasticsearch.client.transport.TransportClient
-import org.elasticsearch.common.settings.ImmutableSettings
-import org.elasticsearch.common.transport.InetSocketTransportAddress
-import org.elasticsearch.transport.ConnectTransportException
-
-class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
- with Logging {
- override val prefix = "ES"
- val client = try {
- val hosts = config.properties.get("HOSTS").
- map(_.split(",").toSeq).getOrElse(Seq("localhost"))
- val ports = config.properties.get("PORTS").
- map(_.split(",").toSeq.map(_.toInt)).getOrElse(Seq(9300))
- val settings = ImmutableSettings.settingsBuilder()
- .put("cluster.name", config.properties.getOrElse("CLUSTERNAME", "elasticsearch"))
- val transportClient = new TransportClient(settings)
- (hosts zip ports) foreach { hp =>
- transportClient.addTransportAddress(
- new InetSocketTransportAddress(hp._1, hp._2))
- }
- transportClient
- } catch {
- case e: ConnectTransportException =>
- throw new StorageClientException(e.getMessage, e)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala b/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
deleted file mode 100644
index 0c549b8..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/package.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-/** Elasticsearch implementation of storage traits, supporting meta data only
- *
- * @group Implementation
- */
-package object elasticsearch {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
deleted file mode 100644
index 2cdb734..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBEventsUtil.scala
+++ /dev/null
@@ -1,415 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase
-
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.EventValidation
-import org.apache.predictionio.data.storage.DataMap
-
-import org.apache.hadoop.hbase.client.Result
-import org.apache.hadoop.hbase.client.Put
-import org.apache.hadoop.hbase.client.Scan
-import org.apache.hadoop.hbase.util.Bytes
-import org.apache.hadoop.hbase.filter.FilterList
-import org.apache.hadoop.hbase.filter.RegexStringComparator
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp
-import org.apache.hadoop.hbase.filter.BinaryComparator
-import org.apache.hadoop.hbase.filter.QualifierFilter
-import org.apache.hadoop.hbase.filter.SkipFilter
-
-import org.json4s.DefaultFormats
-import org.json4s.JObject
-import org.json4s.native.Serialization.{ read, write }
-
-import org.joda.time.DateTime
-import org.joda.time.DateTimeZone
-
-import org.apache.commons.codec.binary.Base64
-import java.security.MessageDigest
-
-import java.util.UUID
-
-/* common utility function for accessing EventsStore in HBase */
-object HBEventsUtil {
-
- implicit val formats = DefaultFormats
-
- def tableName(namespace: String, appId: Int, channelId: Option[Int] = None): String = {
- channelId.map { ch =>
- s"${namespace}:events_${appId}_${ch}"
- }.getOrElse {
- s"${namespace}:events_${appId}"
- }
- }
-
- // column names for "e" column family
- val colNames: Map[String, Array[Byte]] = Map(
- "event" -> "e",
- "entityType" -> "ety",
- "entityId" -> "eid",
- "targetEntityType" -> "tety",
- "targetEntityId" -> "teid",
- "properties" -> "p",
- "prId" -> "prid",
- "eventTime" -> "et",
- "eventTimeZone" -> "etz",
- "creationTime" -> "ct",
- "creationTimeZone" -> "ctz"
- ).mapValues(Bytes.toBytes(_))
-
- def hash(entityType: String, entityId: String): Array[Byte] = {
- val s = entityType + "-" + entityId
- // get a new MessageDigest object each time for thread-safe
- val md5 = MessageDigest.getInstance("MD5")
- md5.digest(Bytes.toBytes(s))
- }
-
- class RowKey(
- val b: Array[Byte]
- ) {
- require((b.size == 32), s"Incorrect b size: ${b.size}")
- lazy val entityHash: Array[Byte] = b.slice(0, 16)
- lazy val millis: Long = Bytes.toLong(b.slice(16, 24))
- lazy val uuidLow: Long = Bytes.toLong(b.slice(24, 32))
-
- lazy val toBytes: Array[Byte] = b
-
- override def toString: String = {
- Base64.encodeBase64URLSafeString(toBytes)
- }
- }
-
- object RowKey {
- def apply(
- entityType: String,
- entityId: String,
- millis: Long,
- uuidLow: Long): RowKey = {
- // add UUID least significant bits for multiple actions at the same time
- // (UUID's most significant bits are actually timestamp,
- // use eventTime instead).
- val b = hash(entityType, entityId) ++
- Bytes.toBytes(millis) ++ Bytes.toBytes(uuidLow)
- new RowKey(b)
- }
-
- // get RowKey from string representation
- def apply(s: String): RowKey = {
- try {
- apply(Base64.decodeBase64(s))
- } catch {
- case e: Exception => throw new RowKeyException(
- s"Failed to convert String ${s} to RowKey because ${e}", e)
- }
- }
-
- def apply(b: Array[Byte]): RowKey = {
- if (b.size != 32) {
- val bString = b.mkString(",")
- throw new RowKeyException(
- s"Incorrect byte array size. Bytes: ${bString}.")
- }
- new RowKey(b)
- }
-
- }
-
- class RowKeyException(val msg: String, val cause: Exception)
- extends Exception(msg, cause) {
- def this(msg: String) = this(msg, null)
- }
-
- case class PartialRowKey(entityType: String, entityId: String,
- millis: Option[Long] = None) {
- val toBytes: Array[Byte] = {
- hash(entityType, entityId) ++
- (millis.map(Bytes.toBytes(_)).getOrElse(Array[Byte]()))
- }
- }
-
- def eventToPut(event: Event, appId: Int): (Put, RowKey) = {
- // generate new rowKey if eventId is None
- val rowKey = event.eventId.map { id =>
- RowKey(id) // create rowKey from eventId
- }.getOrElse {
- // TOOD: use real UUID. not pseudo random
- val uuidLow: Long = UUID.randomUUID().getLeastSignificantBits
- RowKey(
- entityType = event.entityType,
- entityId = event.entityId,
- millis = event.eventTime.getMillis,
- uuidLow = uuidLow
- )
- }
-
- val eBytes = Bytes.toBytes("e")
- // use eventTime as HBase's cell timestamp
- val put = new Put(rowKey.toBytes, event.eventTime.getMillis)
-
- def addStringToE(col: Array[Byte], v: String): Put = {
- put.add(eBytes, col, Bytes.toBytes(v))
- }
-
- def addLongToE(col: Array[Byte], v: Long): Put = {
- put.add(eBytes, col, Bytes.toBytes(v))
- }
-
- addStringToE(colNames("event"), event.event)
- addStringToE(colNames("entityType"), event.entityType)
- addStringToE(colNames("entityId"), event.entityId)
-
- event.targetEntityType.foreach { targetEntityType =>
- addStringToE(colNames("targetEntityType"), targetEntityType)
- }
-
- event.targetEntityId.foreach { targetEntityId =>
- addStringToE(colNames("targetEntityId"), targetEntityId)
- }
-
- // TODO: make properties Option[]
- if (!event.properties.isEmpty) {
- addStringToE(colNames("properties"), write(event.properties.toJObject))
- }
-
- event.prId.foreach { prId =>
- addStringToE(colNames("prId"), prId)
- }
-
- addLongToE(colNames("eventTime"), event.eventTime.getMillis)
- val eventTimeZone = event.eventTime.getZone
- if (!eventTimeZone.equals(EventValidation.defaultTimeZone)) {
- addStringToE(colNames("eventTimeZone"), eventTimeZone.getID)
- }
-
- addLongToE(colNames("creationTime"), event.creationTime.getMillis)
- val creationTimeZone = event.creationTime.getZone
- if (!creationTimeZone.equals(EventValidation.defaultTimeZone)) {
- addStringToE(colNames("creationTimeZone"), creationTimeZone.getID)
- }
-
- // can use zero-length byte array for tag cell value
- (put, rowKey)
- }
-
- def resultToEvent(result: Result, appId: Int): Event = {
- val rowKey = RowKey(result.getRow())
-
- val eBytes = Bytes.toBytes("e")
- // val e = result.getFamilyMap(eBytes)
-
- def getStringCol(col: String): String = {
- val r = result.getValue(eBytes, colNames(col))
- require(r != null,
- s"Failed to get value for column ${col}. " +
- s"Rowkey: ${rowKey.toString} " +
- s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
-
- Bytes.toString(r)
- }
-
- def getLongCol(col: String): Long = {
- val r = result.getValue(eBytes, colNames(col))
- require(r != null,
- s"Failed to get value for column ${col}. " +
- s"Rowkey: ${rowKey.toString} " +
- s"StringBinary: ${Bytes.toStringBinary(result.getRow())}.")
-
- Bytes.toLong(r)
- }
-
- def getOptStringCol(col: String): Option[String] = {
- val r = result.getValue(eBytes, colNames(col))
- if (r == null) {
- None
- } else {
- Some(Bytes.toString(r))
- }
- }
-
- def getTimestamp(col: String): Long = {
- result.getColumnLatestCell(eBytes, colNames(col)).getTimestamp()
- }
-
- val event = getStringCol("event")
- val entityType = getStringCol("entityType")
- val entityId = getStringCol("entityId")
- val targetEntityType = getOptStringCol("targetEntityType")
- val targetEntityId = getOptStringCol("targetEntityId")
- val properties: DataMap = getOptStringCol("properties")
- .map(s => DataMap(read[JObject](s))).getOrElse(DataMap())
- val prId = getOptStringCol("prId")
- val eventTimeZone = getOptStringCol("eventTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
- val eventTime = new DateTime(
- getLongCol("eventTime"), eventTimeZone)
- val creationTimeZone = getOptStringCol("creationTimeZone")
- .map(DateTimeZone.forID(_))
- .getOrElse(EventValidation.defaultTimeZone)
- val creationTime: DateTime = new DateTime(
- getLongCol("creationTime"), creationTimeZone)
-
- Event(
- eventId = Some(RowKey(result.getRow()).toString),
- event = event,
- entityType = entityType,
- entityId = entityId,
- targetEntityType = targetEntityType,
- targetEntityId = targetEntityId,
- properties = properties,
- eventTime = eventTime,
- tags = Seq(),
- prId = prId,
- creationTime = creationTime
- )
- }
-
-
- // for mandatory field. None means don't care.
- // for optional field. None means don't care.
- // Some(None) means not exist.
- // Some(Some(x)) means it should match x
- def createScan(
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None,
- entityType: Option[String] = None,
- entityId: Option[String] = None,
- eventNames: Option[Seq[String]] = None,
- targetEntityType: Option[Option[String]] = None,
- targetEntityId: Option[Option[String]] = None,
- reversed: Option[Boolean] = None): Scan = {
-
- val scan: Scan = new Scan()
-
- (entityType, entityId) match {
- case (Some(et), Some(eid)) => {
- val start = PartialRowKey(et, eid,
- startTime.map(_.getMillis)).toBytes
- // if no untilTime, stop when reach next bytes of entityTypeAndId
- val stop = PartialRowKey(et, eid,
- untilTime.map(_.getMillis).orElse(Some(-1))).toBytes
-
- if (reversed.getOrElse(false)) {
- // Reversed order.
- // If you specify a startRow and stopRow,
- // to scan in reverse, the startRow needs to be lexicographically
- // after the stopRow.
- scan.setStartRow(stop)
- scan.setStopRow(start)
- scan.setReversed(true)
- } else {
- scan.setStartRow(start)
- scan.setStopRow(stop)
- }
- }
- case (_, _) => {
- val minTime: Long = startTime.map(_.getMillis).getOrElse(0)
- val maxTime: Long = untilTime.map(_.getMillis).getOrElse(Long.MaxValue)
- scan.setTimeRange(minTime, maxTime)
- if (reversed.getOrElse(false)) {
- scan.setReversed(true)
- }
- }
- }
-
- val filters = new FilterList(FilterList.Operator.MUST_PASS_ALL)
-
- val eBytes = Bytes.toBytes("e")
-
- def createBinaryFilter(col: String, value: Array[Byte]): SingleColumnValueFilter = {
- val comp = new BinaryComparator(value)
- new SingleColumnValueFilter(
- eBytes, colNames(col), CompareOp.EQUAL, comp)
- }
-
- // skip the row if the column exists
- def createSkipRowIfColumnExistFilter(col: String): SkipFilter = {
- val comp = new BinaryComparator(colNames(col))
- val q = new QualifierFilter(CompareOp.NOT_EQUAL, comp)
- // filters an entire row if any of the Cell checks do not pass
- new SkipFilter(q)
- }
-
- entityType.foreach { et =>
- val compType = new BinaryComparator(Bytes.toBytes(et))
- val filterType = new SingleColumnValueFilter(
- eBytes, colNames("entityType"), CompareOp.EQUAL, compType)
- filters.addFilter(filterType)
- }
-
- entityId.foreach { eid =>
- val compId = new BinaryComparator(Bytes.toBytes(eid))
- val filterId = new SingleColumnValueFilter(
- eBytes, colNames("entityId"), CompareOp.EQUAL, compId)
- filters.addFilter(filterId)
- }
-
- eventNames.foreach { eventsList =>
- // match any of event in the eventsList
- val eventFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE)
- eventsList.foreach { e =>
- val compEvent = new BinaryComparator(Bytes.toBytes(e))
- val filterEvent = new SingleColumnValueFilter(
- eBytes, colNames("event"), CompareOp.EQUAL, compEvent)
- eventFilters.addFilter(filterEvent)
- }
- if (!eventFilters.getFilters().isEmpty) {
- filters.addFilter(eventFilters)
- }
- }
-
- targetEntityType.foreach { tetOpt =>
- if (tetOpt.isEmpty) {
- val filter = createSkipRowIfColumnExistFilter("targetEntityType")
- filters.addFilter(filter)
- } else {
- tetOpt.foreach { tet =>
- val filter = createBinaryFilter(
- "targetEntityType", Bytes.toBytes(tet))
- // the entire row will be skipped if the column is not found.
- filter.setFilterIfMissing(true)
- filters.addFilter(filter)
- }
- }
- }
-
- targetEntityId.foreach { teidOpt =>
- if (teidOpt.isEmpty) {
- val filter = createSkipRowIfColumnExistFilter("targetEntityId")
- filters.addFilter(filter)
- } else {
- teidOpt.foreach { teid =>
- val filter = createBinaryFilter(
- "targetEntityId", Bytes.toBytes(teid))
- // the entire row will be skipped if the column is not found.
- filter.setFilterIfMissing(true)
- filters.addFilter(filter)
- }
- }
- }
-
- if (!filters.getFilters().isEmpty) {
- scan.setFilter(filters)
- }
-
- scan
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
deleted file mode 100644
index 360b007..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBLEvents.scala
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase
-
-import grizzled.slf4j.Logging
-import org.apache.predictionio.data.storage.Event
-import org.apache.predictionio.data.storage.LEvents
-import org.apache.predictionio.data.storage.StorageClientConfig
-import org.apache.predictionio.data.storage.hbase.HBEventsUtil.RowKey
-import org.apache.hadoop.hbase.HColumnDescriptor
-import org.apache.hadoop.hbase.HTableDescriptor
-import org.apache.hadoop.hbase.NamespaceDescriptor
-import org.apache.hadoop.hbase.TableName
-import org.apache.hadoop.hbase.client._
-import org.joda.time.DateTime
-
-import scala.collection.JavaConversions._
-import scala.concurrent.ExecutionContext
-import scala.concurrent.Future
-
-class HBLEvents(val client: HBClient, config: StorageClientConfig, val namespace: String)
- extends LEvents with Logging {
-
- // implicit val formats = DefaultFormats + new EventJson4sSupport.DBSerializer
-
- def resultToEvent(result: Result, appId: Int): Event =
- HBEventsUtil.resultToEvent(result, appId)
-
- def getTable(appId: Int, channelId: Option[Int] = None): HTableInterface =
- client.connection.getTable(HBEventsUtil.tableName(namespace, appId, channelId))
-
- override
- def init(appId: Int, channelId: Option[Int] = None): Boolean = {
- // check namespace exist
- val existingNamespace = client.admin.listNamespaceDescriptors()
- .map(_.getName)
- if (!existingNamespace.contains(namespace)) {
- val nameDesc = NamespaceDescriptor.create(namespace).build()
- info(s"The namespace ${namespace} doesn't exist yet. Creating now...")
- client.admin.createNamespace(nameDesc)
- }
-
- val tableName = TableName.valueOf(HBEventsUtil.tableName(namespace, appId, channelId))
- if (!client.admin.tableExists(tableName)) {
- info(s"The table ${tableName.getNameAsString()} doesn't exist yet." +
- " Creating now...")
- val tableDesc = new HTableDescriptor(tableName)
- tableDesc.addFamily(new HColumnDescriptor("e"))
- tableDesc.addFamily(new HColumnDescriptor("r")) // reserved
- client.admin.createTable(tableDesc)
- }
- true
- }
-
- override
- def remove(appId: Int, channelId: Option[Int] = None): Boolean = {
- val tableName = TableName.valueOf(HBEventsUtil.tableName(namespace, appId, channelId))
- try {
- if (client.admin.tableExists(tableName)) {
- info(s"Removing table ${tableName.getNameAsString()}...")
- client.admin.disableTable(tableName)
- client.admin.deleteTable(tableName)
- } else {
- info(s"Table ${tableName.getNameAsString()} doesn't exist." +
- s" Nothing is deleted.")
- }
- true
- } catch {
- case e: Exception => {
- error(s"Fail to remove table for appId ${appId}. Exception: ${e}")
- false
- }
- }
- }
-
- override
- def close(): Unit = {
- client.admin.close()
- client.connection.close()
- }
-
- override
- def futureInsert(
- event: Event, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
- Future[String] = {
- Future {
- val table = getTable(appId, channelId)
- val (put, rowKey) = HBEventsUtil.eventToPut(event, appId)
- table.put(put)
- table.flushCommits()
- table.close()
- rowKey.toString
- }
- }
-
- override
- def futureGet(
- eventId: String, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
- Future[Option[Event]] = {
- Future {
- val table = getTable(appId, channelId)
- val rowKey = RowKey(eventId)
- val get = new Get(rowKey.toBytes)
-
- val result = table.get(get)
- table.close()
-
- if (!result.isEmpty()) {
- val event = resultToEvent(result, appId)
- Some(event)
- } else {
- None
- }
- }
- }
-
- override
- def futureDelete(
- eventId: String, appId: Int, channelId: Option[Int])(implicit ec: ExecutionContext):
- Future[Boolean] = {
- Future {
- val table = getTable(appId, channelId)
- val rowKey = RowKey(eventId)
- val exists = table.exists(new Get(rowKey.toBytes))
- table.delete(new Delete(rowKey.toBytes))
- table.close()
- exists
- }
- }
-
- override
- def futureFind(
- appId: Int,
- channelId: Option[Int] = None,
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None,
- entityType: Option[String] = None,
- entityId: Option[String] = None,
- eventNames: Option[Seq[String]] = None,
- targetEntityType: Option[Option[String]] = None,
- targetEntityId: Option[Option[String]] = None,
- limit: Option[Int] = None,
- reversed: Option[Boolean] = None)(implicit ec: ExecutionContext):
- Future[Iterator[Event]] = {
- Future {
-
- require(!((reversed == Some(true)) && (entityType.isEmpty || entityId.isEmpty)),
- "the parameter reversed can only be used with both entityType and entityId specified.")
-
- val table = getTable(appId, channelId)
-
- val scan = HBEventsUtil.createScan(
- startTime = startTime,
- untilTime = untilTime,
- entityType = entityType,
- entityId = entityId,
- eventNames = eventNames,
- targetEntityType = targetEntityType,
- targetEntityId = targetEntityId,
- reversed = reversed)
- val scanner = table.getScanner(scan)
- table.close()
-
- val eventsIter = scanner.iterator()
-
- // Get all events if None or Some(-1)
- val results: Iterator[Result] = limit match {
- case Some(-1) => eventsIter
- case None => eventsIter
- case Some(x) => eventsIter.take(x)
- }
-
- val eventsIt = results.map { resultToEvent(_, appId) }
-
- eventsIt
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
deleted file mode 100644
index 7324fa6..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/HBPEvents.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.data.storage.hbase
-
-import org.apache.hadoop.hbase.HBaseConfiguration
-import org.apache.hadoop.hbase.client.{Delete, HTable, Result}
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.hadoop.hbase.mapreduce.{TableInputFormat, TableOutputFormat}
-import org.apache.hadoop.io.Writable
-import org.apache.hadoop.mapreduce.OutputFormat
-import org.apache.predictionio.data.storage.{Event, PEvents, StorageClientConfig}
-import org.apache.spark.SparkContext
-import org.apache.spark.rdd.RDD
-import org.joda.time.DateTime
-
-class HBPEvents(client: HBClient, config: StorageClientConfig, namespace: String) extends PEvents {
-
- def checkTableExists(appId: Int, channelId: Option[Int]): Unit = {
- if (!client.admin.tableExists(HBEventsUtil.tableName(namespace, appId, channelId))) {
- if (channelId.nonEmpty) {
- logger.error(s"The appId $appId with channelId $channelId does not exist." +
- s" Please use valid appId and channelId.")
- throw new Exception(s"HBase table not found for appId $appId" +
- s" with channelId $channelId.")
- } else {
- logger.error(s"The appId $appId does not exist. Please use valid appId.")
- throw new Exception(s"HBase table not found for appId $appId.")
- }
- }
- }
-
- override
- def find(
- appId: Int,
- channelId: Option[Int] = None,
- startTime: Option[DateTime] = None,
- untilTime: Option[DateTime] = None,
- entityType: Option[String] = None,
- entityId: Option[String] = None,
- eventNames: Option[Seq[String]] = None,
- targetEntityType: Option[Option[String]] = None,
- targetEntityId: Option[Option[String]] = None
- )(sc: SparkContext): RDD[Event] = {
-
- checkTableExists(appId, channelId)
-
- val conf = HBaseConfiguration.create()
- conf.set(TableInputFormat.INPUT_TABLE,
- HBEventsUtil.tableName(namespace, appId, channelId))
-
- val scan = HBEventsUtil.createScan(
- startTime = startTime,
- untilTime = untilTime,
- entityType = entityType,
- entityId = entityId,
- eventNames = eventNames,
- targetEntityType = targetEntityType,
- targetEntityId = targetEntityId,
- reversed = None)
- scan.setCaching(500) // TODO
- scan.setCacheBlocks(false) // TODO
-
- conf.set(TableInputFormat.SCAN, PIOHBaseUtil.convertScanToString(scan))
-
- // HBase is not accessed until this rdd is actually used.
- val rdd = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
- classOf[ImmutableBytesWritable],
- classOf[Result]).map {
- case (key, row) => HBEventsUtil.resultToEvent(row, appId)
- }
-
- rdd
- }
-
- override
- def write(
- events: RDD[Event], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
-
- checkTableExists(appId, channelId)
-
- val conf = HBaseConfiguration.create()
- conf.set(TableOutputFormat.OUTPUT_TABLE,
- HBEventsUtil.tableName(namespace, appId, channelId))
- conf.setClass("mapreduce.outputformat.class",
- classOf[TableOutputFormat[Object]],
- classOf[OutputFormat[Object, Writable]])
-
- events.map { event =>
- val (put, rowKey) = HBEventsUtil.eventToPut(event, appId)
- (new ImmutableBytesWritable(rowKey.toBytes), put)
- }.saveAsNewAPIHadoopDataset(conf)
-
- }
-
- def delete(
- eventIds: RDD[String], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
-
- checkTableExists(appId, channelId)
-
- val tableName = HBEventsUtil.tableName(namespace, appId, channelId)
-
- eventIds.foreachPartition{ iter =>
- val conf = HBaseConfiguration.create()
- conf.set(TableOutputFormat.OUTPUT_TABLE,
- tableName)
-
- val table = new HTable(conf, tableName)
- iter.foreach { id =>
- val rowKey = HBEventsUtil.RowKey(id)
- val delete = new Delete(rowKey.b)
- table.delete(delete)
- }
- table.close
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
deleted file mode 100644
index 745fcb9..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/PIOHBaseUtil.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.predictionio.data.storage.hbase
-
-import org.apache.hadoop.hbase.client.Scan
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil
-import org.apache.hadoop.hbase.util.Base64
-
-object PIOHBaseUtil {
- /*
- * Copying this from Apache HBase because of its restrictive scope in 0.98.x
- */
- def convertScanToString(scan: Scan): String = {
- val proto = ProtobufUtil.toScan(scan)
- Base64.encodeBytes(proto.toByteArray)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
deleted file mode 100644
index 1720410..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/StorageClient.scala
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage.hbase
-
-import org.apache.predictionio.data.storage.BaseStorageClient
-import org.apache.predictionio.data.storage.StorageClientConfig
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.HBaseConfiguration
-import org.apache.hadoop.hbase.MasterNotRunningException
-import org.apache.hadoop.hbase.ZooKeeperConnectionException
-import org.apache.hadoop.hbase.client.HConnectionManager
-import org.apache.hadoop.hbase.client.HConnection
-import org.apache.hadoop.hbase.client.HBaseAdmin
-
-import grizzled.slf4j.Logging
-
-case class HBClient(
- val conf: Configuration,
- val connection: HConnection,
- val admin: HBaseAdmin
-)
-
-class StorageClient(val config: StorageClientConfig)
- extends BaseStorageClient with Logging {
-
- val conf = HBaseConfiguration.create()
-
- if (config.test) {
- // use fewer retries and shorter timeout for test mode
- conf.set("hbase.client.retries.number", "1")
- conf.set("zookeeper.session.timeout", "30000");
- conf.set("zookeeper.recovery.retry", "1")
- }
-
- try {
- HBaseAdmin.checkHBaseAvailable(conf)
- } catch {
- case e: MasterNotRunningException =>
- error("HBase master is not running (ZooKeeper ensemble: " +
- conf.get("hbase.zookeeper.quorum") + "). Please make sure that HBase " +
- "is running properly, and that the configuration is pointing at the " +
- "correct ZooKeeper ensemble.")
- throw e
- case e: ZooKeeperConnectionException =>
- error("Cannot connect to ZooKeeper (ZooKeeper ensemble: " +
- conf.get("hbase.zookeeper.quorum") + "). Please make sure that the " +
- "configuration is pointing at the correct ZooKeeper ensemble. By " +
- "default, HBase manages its own ZooKeeper, so if you have not " +
- "configured HBase to use an external ZooKeeper, that means your " +
- "HBase is not started or configured properly.")
- throw e
- case e: Exception => {
- error("Failed to connect to HBase." +
- " Please check if HBase is running properly.")
- throw e
- }
- }
-
- val connection = HConnectionManager.createConnection(conf)
-
- val client = HBClient(
- conf = conf,
- connection = connection,
- admin = new HBaseAdmin(connection)
- )
-
- override
- val prefix = "HB"
-}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/data/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
----------------------------------------------------------------------
diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala b/data/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
deleted file mode 100644
index 49bf031..0000000
--- a/data/src/main/scala/org/apache/predictionio/data/storage/hbase/package.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.predictionio.data.storage
-
-/** HBase implementation of storage traits, supporting event data only
- *
- * @group Implementation
- */
-package object hbase {}
[19/50] [abbrv] incubator-predictionio git commit: Limit
Elasticsearch 5 JVM memory usage
Posted by do...@apache.org.
Limit Elasticsearch 5 JVM memory usage
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/2da30b06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/2da30b06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/2da30b06
Branch: refs/heads/livedoc
Commit: 2da30b06eb7d46ad351015dc5f3067f37d9fb327
Parents: 57aa82f
Author: Donald Szeto <do...@apache.org>
Authored: Fri Mar 10 23:38:37 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Fri Mar 10 23:38:37 2017 -0800
----------------------------------------------------------------------
tests/docker-compose.yml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/2da30b06/tests/docker-compose.yml
----------------------------------------------------------------------
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
index b6774d3..06126a5 100644
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -19,6 +19,7 @@ services:
image: docker.elastic.co/elasticsearch/elasticsearch:5.2.2
environment:
- xpack.security.enabled=false
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
hbase:
image: harisekhon/hbase:1.0
postgres:
[27/50] [abbrv] incubator-predictionio git commit: Unit test script
should fail immediately
Posted by do...@apache.org.
Unit test script should fail immediately
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/606be414
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/606be414
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/606be414
Branch: refs/heads/livedoc
Commit: 606be4148fe7d4253193a4ef3c4e996d27c0ad51
Parents: ae51040
Author: Donald Szeto <do...@apache.org>
Authored: Wed Mar 15 09:40:26 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Wed Mar 15 09:40:26 2017 -0700
----------------------------------------------------------------------
tests/unit.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/606be414/tests/unit.sh
----------------------------------------------------------------------
diff --git a/tests/unit.sh b/tests/unit.sh
index 324b87f..6382a70 100755
--- a/tests/unit.sh
+++ b/tests/unit.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -ex
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
[47/50] [abbrv] incubator-predictionio git commit: Properly combine
spark-submit arguments
Posted by do...@apache.org.
Properly combine spark-submit arguments
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/9deca1a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/9deca1a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/9deca1a4
Branch: refs/heads/livedoc
Commit: 9deca1a47fccb6c58fba366d6dd1c8bf751b1cf9
Parents: 070f179
Author: Donald Szeto <do...@apache.org>
Authored: Sat Apr 8 22:48:00 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sat Apr 8 22:48:00 2017 -0700
----------------------------------------------------------------------
.../org/apache/predictionio/tools/Runner.scala | 101 ++++++++++++++++++-
.../apache/predictionio/tools/RunnerSpec.scala | 73 ++++++++++++++
2 files changed, 170 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/9deca1a4/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index 70e3837..4a721be 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.predictionio.tools.ReturnTypes._
import org.apache.predictionio.workflow.WorkflowUtils
+import scala.collection.mutable
import scala.sys.process._
case class SparkArgs(
@@ -95,6 +96,92 @@ object Runner extends EitherLogging {
}
}
+ /** Group argument values by argument names
+ *
+ * This only works with long argument names immediately followed by a value
+ *
+ * Input:
+ * Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ *
+ * Output:
+ * Map("--foo" -> Seq("bar", "jeez"), "--dead"- > "beef baz")
+ *
+ * @param arguments Sequence of argument names and values
+ * @return A map with argument values keyed by the same argument name
+ */
+ def groupByArgumentName(arguments: Seq[String]): Map[String, Seq[String]] = {
+ val argumentMap = mutable.HashMap.empty[String, Seq[String]]
+ arguments.foldLeft("") { (prev, current) =>
+ if (prev.startsWith("--") && !current.startsWith("--")) {
+ if (argumentMap.contains(prev)) {
+ argumentMap(prev) = argumentMap(prev) :+ current
+ } else {
+ argumentMap(prev) = Seq(current)
+ }
+ }
+ current
+ }
+ argumentMap.toMap
+ }
+
+ /** Remove argument names and values
+ *
+ * This only works with long argument names immediately followed by a value
+ *
+ * Input:
+ * Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ * Set("--flag", "--foo")
+ *
+ * Output:
+ * Seq("--flag", "--dead", "beef baz", "n00b")
+ *
+ * @param arguments Sequence of argument names and values
+ * @param remove Name of argument and associated values to remove
+ * @return Sequence of argument names and values with targets removed
+ */
+ def removeArguments(arguments: Seq[String], remove: Set[String]): Seq[String] = {
+ if (remove.isEmpty) {
+ arguments
+ } else {
+ arguments.foldLeft(Seq.empty[String]) { (ongoing, current) =>
+ if (ongoing.isEmpty) {
+ Seq(current)
+ } else {
+ if (remove.contains(ongoing.last) && !current.startsWith("--")) {
+ ongoing.take(ongoing.length - 1)
+ } else {
+ ongoing :+ current
+ }
+ }
+ }
+ }
+ }
+
+ /** Combine repeated arguments together
+ *
+ * Input:
+ * Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ * Map("--foo", (_ + _))
+ *
+ * Output:
+ * Seq("--flag", "--dead", "beef baz", "n00b", "--foo", "bar jeez")
+ *
+ * @param arguments Sequence of argument names and values
+ * @param combinators Map of argument name to combinator function
+ * @return Sequence of argument names and values with specific argument values combined
+ */
+ def combineArguments(
+ arguments: Seq[String],
+ combinators: Map[String, (String, String) => String]): Seq[String] = {
+ val argumentsToCombine: Map[String, Seq[String]] =
+ groupByArgumentName(arguments).filterKeys(combinators.keySet.contains(_))
+ val argumentsMinusToCombine = removeArguments(arguments, combinators.keySet)
+ val combinedArguments = argumentsToCombine flatMap { kv =>
+ Seq(kv._1, kv._2.reduce(combinators(kv._1)))
+ }
+ argumentsMinusToCombine ++ combinedArguments
+ }
+
def runOnSpark(
className: String,
classArgs: Seq[String],
@@ -189,17 +276,23 @@ object Runner extends EitherLogging {
}
val verboseArg = if (verbose) Seq("--verbose") else Nil
- val pioLogDir = Option(System.getProperty("pio.log.dir")).getOrElse(s"${pioHome}/log")
+ val pioLogDir = Option(System.getProperty("pio.log.dir")).getOrElse(s"$pioHome/log")
- val sparkSubmit = Seq(
- sparkSubmitCommand,
+ val sparkSubmitArgs = Seq(
sa.sparkPassThrough,
Seq("--class", className),
sparkSubmitJars,
sparkSubmitFiles,
sparkSubmitExtraClasspaths,
sparkSubmitKryo,
- Seq("--driver-java-options", s"-Dpio.log.dir=${pioLogDir}"),
+ Seq("--driver-java-options", s"-Dpio.log.dir=$pioLogDir")).flatten
+
+ val whitespaceCombinator = (a: String, b: String) => s"$a $b"
+ val combinators = Map("--driver-java-options" -> whitespaceCombinator)
+
+ val sparkSubmit = Seq(
+ sparkSubmitCommand,
+ combineArguments(sparkSubmitArgs, combinators),
Seq(mainJar),
detectFilePaths(fs, sa.scratchUri, classArgs),
Seq("--env", pioEnvVars),
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/9deca1a4/tools/src/test/scala/org/apache/predictionio/tools/RunnerSpec.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/predictionio/tools/RunnerSpec.scala b/tools/src/test/scala/org/apache/predictionio/tools/RunnerSpec.scala
new file mode 100644
index 0000000..92317f8
--- /dev/null
+++ b/tools/src/test/scala/org/apache/predictionio/tools/RunnerSpec.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.predictionio.tools
+
+import org.specs2.mutable.Specification
+
+class RunnerSpec extends Specification {
+ "groupByArgumentName" >> {
+ "test1" >> {
+ val test = Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ Runner.groupByArgumentName(test) must havePairs(
+ "--foo" -> Seq("bar", "jeez"),
+ "--dead" -> Seq("beef baz"))
+ }
+
+ "test2" >> {
+ val test =
+ Seq("--foo", "--bar", "flag", "--dead", "beef baz", "n00b", "--foo", "jeez", "--flag")
+ Runner.groupByArgumentName(test) must havePairs(
+ "--foo" -> Seq("jeez"),
+ "--bar" -> Seq("flag"),
+ "--dead" -> Seq("beef baz"))
+ }
+ }
+
+ "removeArguments" >> {
+ "test1" >> {
+ val test = Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ val remove = Set("--flag", "--foo")
+ Runner.removeArguments(test, remove) === Seq("--flag", "--dead", "beef baz", "n00b")
+ }
+
+ "test2" >> {
+ val test =
+ Seq("--foo", "--bar", "flag", "--dead", "beef baz", "n00b", "--foo", "jeez", "--flag")
+ val remove = Set("--flag", "--foo")
+ Runner.removeArguments(test, remove) ===
+ Seq("--foo", "--bar", "flag", "--dead", "beef baz", "n00b", "--flag")
+ }
+ }
+
+ "combineArguments" >> {
+ "test1" >> {
+ val test = Seq("--foo", "bar", "--flag", "--dead", "beef baz", "n00b", "--foo", "jeez")
+ val combinators = Map("--foo" -> ((a: String, b: String) => s"$a $b"))
+ Runner.combineArguments(test, combinators) ===
+ Seq("--flag", "--dead", "beef baz", "n00b", "--foo", "bar jeez")
+ }
+
+ "test2" >> {
+ val test =
+ Seq("--foo", "--bar", "flag", "--dead", "beef baz", "n00b", "--foo", "jeez", "--flag")
+ val combinators = Map("--foo" -> ((a: String, b: String) => s"$a $b"))
+ Runner.combineArguments(test, combinators) ===
+ Seq("--foo", "--bar", "flag", "--dead", "beef baz", "n00b", "--flag", "--foo", "jeez")
+ }
+ }
+}
[11/50] [abbrv] incubator-predictionio git commit: [PIO-49] Add
support for Elasticsearch 5
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
new file mode 100644
index 0000000..de74d46
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/storage/hbase/upgrade/Upgrade_0_8_3.scala
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hbase.upgrade
+
+import org.apache.predictionio.annotation.Experimental
+
+import grizzled.slf4j.Logger
+import org.apache.predictionio.data.storage.Storage
+import org.apache.predictionio.data.storage.DataMap
+import org.apache.predictionio.data.storage.hbase.HBLEvents
+import org.apache.predictionio.data.storage.hbase.HBEventsUtil
+
+import scala.collection.JavaConversions._
+
+import scala.concurrent._
+import ExecutionContext.Implicits.global
+import org.apache.predictionio.data.storage.LEvents
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+import java.lang.Thread
+
+object CheckDistribution {
+ def entityType(eventClient: LEvents, appId: Int)
+ : Map[(String, Option[String]), Int] = {
+ eventClient
+ .find(appId = appId)
+ .foldLeft(Map[(String, Option[String]), Int]().withDefaultValue(0)) {
+ case (m, e) => {
+ val k = (e.entityType, e.targetEntityType)
+ m.updated(k, m(k) + 1)
+ }
+ }
+ }
+
+ def runMain(appId: Int) {
+ val eventClient = Storage.getLEvents().asInstanceOf[HBLEvents]
+
+ entityType(eventClient, appId)
+ .toSeq
+ .sortBy(-_._2)
+ .foreach { println }
+
+ }
+
+ def main(args: Array[String]) {
+ runMain(args(0).toInt)
+ }
+
+}
+
+/** :: Experimental :: */
+@Experimental
+object Upgrade_0_8_3 {
+ val NameMap = Map(
+ "pio_user" -> "user",
+ "pio_item" -> "item")
+ val RevNameMap = NameMap.toSeq.map(_.swap).toMap
+
+ val logger = Logger[this.type]
+
+ def main(args: Array[String]) {
+ val fromAppId = args(0).toInt
+ val toAppId = args(1).toInt
+
+ runMain(fromAppId, toAppId)
+ }
+
+ def runMain(fromAppId: Int, toAppId: Int): Unit = {
+ upgrade(fromAppId, toAppId)
+ }
+
+
+ val obsEntityTypes = Set("pio_user", "pio_item")
+ val obsProperties = Set(
+ "pio_itypes", "pio_starttime", "pio_endtime",
+ "pio_inactive", "pio_price", "pio_rating")
+
+ def hasPIOPrefix(eventClient: LEvents, appId: Int): Boolean = {
+ eventClient.find(appId = appId).filter( e =>
+ (obsEntityTypes.contains(e.entityType) ||
+ e.targetEntityType.map(obsEntityTypes.contains(_)).getOrElse(false) ||
+ (!e.properties.keySet.forall(!obsProperties.contains(_)))
+ )
+ ).hasNext
+ }
+
+ def isEmpty(eventClient: LEvents, appId: Int): Boolean =
+ !eventClient.find(appId = appId).hasNext
+
+
+ def upgradeCopy(eventClient: LEvents, fromAppId: Int, toAppId: Int) {
+ val fromDist = CheckDistribution.entityType(eventClient, fromAppId)
+
+ logger.info("FromAppId Distribution")
+ fromDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
+
+ val events = eventClient
+ .find(appId = fromAppId)
+ .zipWithIndex
+ .foreach { case (fromEvent, index) => {
+ if (index % 50000 == 0) {
+ // logger.info(s"Progress: $fromEvent $index")
+ logger.info(s"Progress: $index")
+ }
+
+
+ val fromEntityType = fromEvent.entityType
+ val toEntityType = NameMap.getOrElse(fromEntityType, fromEntityType)
+
+ val fromTargetEntityType = fromEvent.targetEntityType
+ val toTargetEntityType = fromTargetEntityType
+ .map { et => NameMap.getOrElse(et, et) }
+
+ val toProperties = DataMap(fromEvent.properties.fields.map {
+ case (k, v) =>
+ val newK = if (obsProperties.contains(k)) {
+ val nK = k.stripPrefix("pio_")
+ logger.info(s"property ${k} will be renamed to ${nK}")
+ nK
+ } else k
+ (newK, v)
+ })
+
+ val toEvent = fromEvent.copy(
+ entityType = toEntityType,
+ targetEntityType = toTargetEntityType,
+ properties = toProperties)
+
+ eventClient.insert(toEvent, toAppId)
+ }}
+
+
+ val toDist = CheckDistribution.entityType(eventClient, toAppId)
+
+ logger.info("Recap fromAppId Distribution")
+ fromDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
+
+ logger.info("ToAppId Distribution")
+ toDist.toSeq.sortBy(-_._2).foreach { e => logger.info(e) }
+
+ val fromGood = fromDist
+ .toSeq
+ .forall { case (k, c) => {
+ val (et, tet) = k
+ val net = NameMap.getOrElse(et, et)
+ val ntet = tet.map(tet => NameMap.getOrElse(tet, tet))
+ val nk = (net, ntet)
+ val nc = toDist.getOrElse(nk, -1)
+ val checkMatch = (c == nc)
+ if (!checkMatch) {
+ logger.info(s"${k} doesn't match: old has ${c}. new has ${nc}.")
+ }
+ checkMatch
+ }}
+
+ val toGood = toDist
+ .toSeq
+ .forall { case (k, c) => {
+ val (et, tet) = k
+ val oet = RevNameMap.getOrElse(et, et)
+ val otet = tet.map(tet => RevNameMap.getOrElse(tet, tet))
+ val ok = (oet, otet)
+ val oc = fromDist.getOrElse(ok, -1)
+ val checkMatch = (c == oc)
+ if (!checkMatch) {
+ logger.info(s"${k} doesn't match: new has ${c}. old has ${oc}.")
+ }
+ checkMatch
+ }}
+
+ if (!fromGood || !toGood) {
+ logger.error("Doesn't match!! There is an import error.")
+ } else {
+ logger.info("Count matches. Looks like we are good to go.")
+ }
+ }
+
+ /* For upgrade from 0.8.2 to 0.8.3 only */
+ def upgrade(fromAppId: Int, toAppId: Int) {
+
+ val eventClient = Storage.getLEvents().asInstanceOf[HBLEvents]
+
+ require(fromAppId != toAppId,
+ s"FromAppId: $fromAppId must be different from toAppId: $toAppId")
+
+ if (hasPIOPrefix(eventClient, fromAppId)) {
+ require(
+ isEmpty(eventClient, toAppId),
+ s"Target appId: $toAppId is not empty. Please run " +
+ "`pio app data-delete <app_name>` to clean the data before upgrading")
+
+ logger.info(s"$fromAppId isEmpty: " + isEmpty(eventClient, fromAppId))
+
+ upgradeCopy(eventClient, fromAppId, toAppId)
+
+ } else {
+ logger.info(s"From appId: ${fromAppId} doesn't contain"
+ + s" obsolete entityTypes ${obsEntityTypes} or"
+ + s" obsolete properties ${obsProperties}."
+ + " No need data migration."
+ + s" You can continue to use appId ${fromAppId}.")
+ }
+
+ logger.info("Done.")
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
----------------------------------------------------------------------
diff --git a/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala b/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
new file mode 100644
index 0000000..b453820
--- /dev/null
+++ b/storage/hbase/src/main/scala/org/apache/predictionio/data/view/PBatchView.scala
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.view
+
+import org.apache.predictionio.data.storage.hbase.HBPEvents
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.EventValidation
+import org.apache.predictionio.data.storage.DataMap
+import org.apache.predictionio.data.storage.Storage
+
+import org.joda.time.DateTime
+
+import org.json4s.JValue
+
+import org.apache.spark.SparkContext
+import org.apache.spark.SparkContext._
+import org.apache.spark.rdd.RDD
+
+
+// each JValue data associated with the time it is set
+private[predictionio] case class PropTime(val d: JValue, val t: Long) extends Serializable
+
+private[predictionio] case class SetProp (
+ val fields: Map[String, PropTime],
+ // last set time. Note: fields could be empty with valid set time
+ val t: Long) extends Serializable {
+
+ def ++ (that: SetProp): SetProp = {
+ val commonKeys = fields.keySet.intersect(that.fields.keySet)
+
+ val common: Map[String, PropTime] = commonKeys.map { k =>
+ val thisData = this.fields(k)
+ val thatData = that.fields(k)
+ // only keep the value with latest time
+ val v = if (thisData.t > thatData.t) thisData else thatData
+ (k, v)
+ }.toMap
+
+ val combinedFields = common ++
+ (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
+
+ // keep the latest set time
+ val combinedT = if (this.t > that.t) this.t else that.t
+
+ SetProp(
+ fields = combinedFields,
+ t = combinedT
+ )
+ }
+}
+
+private[predictionio] case class UnsetProp (fields: Map[String, Long]) extends Serializable {
+ def ++ (that: UnsetProp): UnsetProp = {
+ val commonKeys = fields.keySet.intersect(that.fields.keySet)
+
+ val common: Map[String, Long] = commonKeys.map { k =>
+ val thisData = this.fields(k)
+ val thatData = that.fields(k)
+ // only keep the value with latest time
+ val v = if (thisData > thatData) thisData else thatData
+ (k, v)
+ }.toMap
+
+ val combinedFields = common ++
+ (this.fields -- commonKeys) ++ (that.fields -- commonKeys)
+
+ UnsetProp(
+ fields = combinedFields
+ )
+ }
+}
+
+private[predictionio] case class DeleteEntity (t: Long) extends Serializable {
+ def ++ (that: DeleteEntity): DeleteEntity = {
+ if (this.t > that.t) this else that
+ }
+}
+
+private[predictionio] case class EventOp (
+ val setProp: Option[SetProp] = None,
+ val unsetProp: Option[UnsetProp] = None,
+ val deleteEntity: Option[DeleteEntity] = None
+) extends Serializable {
+
+ def ++ (that: EventOp): EventOp = {
+ EventOp(
+ setProp = (setProp ++ that.setProp).reduceOption(_ ++ _),
+ unsetProp = (unsetProp ++ that.unsetProp).reduceOption(_ ++ _),
+ deleteEntity = (deleteEntity ++ that.deleteEntity).reduceOption(_ ++ _)
+ )
+ }
+
+ def toDataMap(): Option[DataMap] = {
+ setProp.flatMap { set =>
+
+ val unsetKeys: Set[String] = unsetProp.map( unset =>
+ unset.fields.filter{ case (k, v) => (v >= set.fields(k).t) }.keySet
+ ).getOrElse(Set())
+
+ val combinedFields = deleteEntity.map { delete =>
+ if (delete.t >= set.t) {
+ None
+ } else {
+ val deleteKeys: Set[String] = set.fields
+ .filter { case (k, PropTime(kv, t)) =>
+ (delete.t >= t)
+ }.keySet
+ Some(set.fields -- unsetKeys -- deleteKeys)
+ }
+ }.getOrElse{
+ Some(set.fields -- unsetKeys)
+ }
+
+ // Note: mapValues() doesn't return concrete Map and causes
+ // NotSerializableException issue. Use map(identity) to work around this.
+ // see https://issues.scala-lang.org/browse/SI-7005
+ combinedFields.map(f => DataMap(f.mapValues(_.d).map(identity)))
+ }
+ }
+
+}
+
+private[predictionio] object EventOp {
+ def apply(e: Event): EventOp = {
+ val t = e.eventTime.getMillis
+ e.event match {
+ case "$set" => {
+ val fields = e.properties.fields.mapValues(jv =>
+ PropTime(jv, t)
+ ).map(identity)
+
+ EventOp(
+ setProp = Some(SetProp(fields = fields, t = t))
+ )
+ }
+ case "$unset" => {
+ val fields = e.properties.fields.mapValues(jv => t).map(identity)
+ EventOp(
+ unsetProp = Some(UnsetProp(fields = fields))
+ )
+ }
+ case "$delete" => {
+ EventOp(
+ deleteEntity = Some(DeleteEntity(t))
+ )
+ }
+ case _ => {
+ EventOp()
+ }
+ }
+ }
+}
+
+@deprecated("Use PEvents or PEventStore instead.", "0.9.2")
+class PBatchView(
+ val appId: Int,
+ val startTime: Option[DateTime],
+ val untilTime: Option[DateTime],
+ val sc: SparkContext) {
+
+ // NOTE: parallel Events DB interface
+ @transient lazy val eventsDb = Storage.getPEvents()
+
+ @transient lazy val _events: RDD[Event] =
+ eventsDb.getByAppIdAndTimeAndEntity(
+ appId = appId,
+ startTime = startTime,
+ untilTime = untilTime,
+ entityType = None,
+ entityId = None)(sc)
+
+ // TODO: change to use EventSeq?
+ @transient lazy val events: RDD[Event] = _events
+
+ def aggregateProperties(
+ entityType: String,
+ startTimeOpt: Option[DateTime] = None,
+ untilTimeOpt: Option[DateTime] = None
+ ): RDD[(String, DataMap)] = {
+
+ _events
+ .filter( e => ((e.entityType == entityType) &&
+ (EventValidation.isSpecialEvents(e.event))) )
+ .map( e => (e.entityId, EventOp(e) ))
+ .aggregateByKey[EventOp](EventOp())(
+ // within same partition
+ seqOp = { case (u, v) => u ++ v },
+ // across partition
+ combOp = { case (accu, u) => accu ++ u }
+ )
+ .mapValues(_.toDataMap)
+ .filter{ case (k, v) => v.isDefined }
+ .map{ case (k, v) => (k, v.get) }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hbase/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/hbase/src/test/resources/application.conf b/storage/hbase/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/hbase/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/.gitignore
----------------------------------------------------------------------
diff --git a/storage/hdfs/.gitignore b/storage/hdfs/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/hdfs/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/hdfs/build.sbt b/storage/hdfs/build.sbt
new file mode 100644
index 0000000..9f064c6
--- /dev/null
+++ b/storage/hdfs/build.sbt
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-hdfs"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-hdfs-assembly-" + version.value + ".jar")
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
----------------------------------------------------------------------
diff --git a/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
new file mode 100644
index 0000000..08dfb01
--- /dev/null
+++ b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/HDFSModels.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hdfs
+
+import java.io.IOException
+
+import com.google.common.io.ByteStreams
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Model
+import org.apache.predictionio.data.storage.Models
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+
+class HDFSModels(fs: FileSystem, config: StorageClientConfig, prefix: String)
+ extends Models with Logging {
+
+ def insert(i: Model): Unit = {
+ try {
+ val fsdos = fs.create(new Path(s"$prefix${i.id}"))
+ fsdos.write(i.models)
+ fsdos.close
+ } catch {
+ case e: IOException => error(e.getMessage)
+ }
+ }
+
+ def get(id: String): Option[Model] = {
+ try {
+ val p = new Path(s"$prefix$id")
+ Some(Model(
+ id = id,
+ models = ByteStreams.toByteArray(fs.open(p))))
+ } catch {
+ case e: Throwable =>
+ error(e.getMessage)
+ None
+ }
+ }
+
+ def delete(id: String): Unit = {
+ val p = new Path(s"$prefix$id")
+ if (!fs.delete(p, false)) {
+ error(s"Unable to delete ${fs.makeQualified(p).toString}!")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
new file mode 100644
index 0000000..bc57f2a
--- /dev/null
+++ b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/StorageClient.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.hdfs
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem
+import org.apache.hadoop.fs.Path
+
+class StorageClient(val config: StorageClientConfig) extends BaseStorageClient
+ with Logging {
+ override val prefix = "HDFS"
+ val conf = new Configuration
+ val fs = FileSystem.get(conf)
+ fs.setWorkingDirectory(
+ new Path(config.properties.getOrElse("PATH", config.properties("HOSTS"))))
+ val client = fs
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
----------------------------------------------------------------------
diff --git a/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
new file mode 100644
index 0000000..a927d78
--- /dev/null
+++ b/storage/hdfs/src/main/scala/org/apache/predictionio/data/storage/hdfs/package.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** HDFS implementation of storage traits, supporting model data only
+ *
+ * @group Implementation
+ */
+package object hdfs {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/hdfs/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/hdfs/src/test/resources/application.conf b/storage/hdfs/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/hdfs/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/.gitignore
----------------------------------------------------------------------
diff --git a/storage/jdbc/.gitignore b/storage/jdbc/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/jdbc/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/build.sbt
----------------------------------------------------------------------
diff --git a/storage/jdbc/build.sbt b/storage/jdbc/build.sbt
new file mode 100644
index 0000000..63d420b
--- /dev/null
+++ b/storage/jdbc/build.sbt
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-jdbc"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
+ "org.postgresql" % "postgresql" % "9.4-1204-jdbc41",
+ "org.scalikejdbc" %% "scalikejdbc" % "2.3.5",
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-jdbc-assembly-" + version.value + ".jar")
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
new file mode 100644
index 0000000..437f8ae
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCAccessKeys.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.AccessKey
+import org.apache.predictionio.data.storage.AccessKeys
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+import scala.util.Random
+
+/** JDBC implementation of [[AccessKeys]] */
+class JDBCAccessKeys(client: String, config: StorageClientConfig, prefix: String)
+ extends AccessKeys with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "accesskeys")
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ accesskey varchar(64) not null primary key,
+ appid integer not null,
+ events text)""".execute().apply()
+ }
+
+ def insert(accessKey: AccessKey): Option[String] = DB localTx { implicit s =>
+ val key = if (accessKey.key.isEmpty) generateKey else accessKey.key
+ val events = if (accessKey.events.isEmpty) None else Some(accessKey.events.mkString(","))
+ sql"""
+ insert into $tableName values(
+ $key,
+ ${accessKey.appid},
+ $events)""".update().apply()
+ Some(key)
+ }
+
+ def get(key: String): Option[AccessKey] = DB readOnly { implicit session =>
+ sql"SELECT accesskey, appid, events FROM $tableName WHERE accesskey = $key".
+ map(resultToAccessKey).single().apply()
+ }
+
+ def getAll(): Seq[AccessKey] = DB readOnly { implicit session =>
+ sql"SELECT accesskey, appid, events FROM $tableName".map(resultToAccessKey).list().apply()
+ }
+
+ def getByAppid(appid: Int): Seq[AccessKey] = DB readOnly { implicit session =>
+ sql"SELECT accesskey, appid, events FROM $tableName WHERE appid = $appid".
+ map(resultToAccessKey).list().apply()
+ }
+
+ def update(accessKey: AccessKey): Unit = DB localTx { implicit session =>
+ val events = if (accessKey.events.isEmpty) None else Some(accessKey.events.mkString(","))
+ sql"""
+ UPDATE $tableName SET
+ appid = ${accessKey.appid},
+ events = $events
+ WHERE accesskey = ${accessKey.key}""".update().apply()
+ }
+
+ def delete(key: String): Unit = DB localTx { implicit session =>
+ sql"DELETE FROM $tableName WHERE accesskey = $key".update().apply()
+ }
+
+ /** Convert JDBC results to [[AccessKey]] */
+ def resultToAccessKey(rs: WrappedResultSet): AccessKey = {
+ AccessKey(
+ key = rs.string("accesskey"),
+ appid = rs.int("appid"),
+ events = rs.stringOpt("events").map(_.split(",").toSeq).getOrElse(Nil))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
new file mode 100644
index 0000000..17e6410
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCApps.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.App
+import org.apache.predictionio.data.storage.Apps
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+/** JDBC implementation of [[Apps]] */
+class JDBCApps(client: String, config: StorageClientConfig, prefix: String)
+ extends Apps with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "apps")
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ id serial not null primary key,
+ name text not null,
+ description text)""".execute.apply()
+ }
+
+ def insert(app: App): Option[Int] = DB localTx { implicit session =>
+ val q = if (app.id == 0) {
+ sql"""
+ insert into $tableName (name, description) values(${app.name}, ${app.description})
+ """
+ } else {
+ sql"""
+ insert into $tableName values(${app.id}, ${app.name}, ${app.description})
+ """
+ }
+ Some(q.updateAndReturnGeneratedKey().apply().toInt)
+ }
+
+ def get(id: Int): Option[App] = DB readOnly { implicit session =>
+ sql"SELECT id, name, description FROM $tableName WHERE id = ${id}".map(rs =>
+ App(
+ id = rs.int("id"),
+ name = rs.string("name"),
+ description = rs.stringOpt("description"))
+ ).single().apply()
+ }
+
+ def getByName(name: String): Option[App] = DB readOnly { implicit session =>
+ sql"SELECT id, name, description FROM $tableName WHERE name = ${name}".map(rs =>
+ App(
+ id = rs.int("id"),
+ name = rs.string("name"),
+ description = rs.stringOpt("description"))
+ ).single().apply()
+ }
+
+ def getAll(): Seq[App] = DB readOnly { implicit session =>
+ sql"SELECT id, name, description FROM $tableName".map(rs =>
+ App(
+ id = rs.int("id"),
+ name = rs.string("name"),
+ description = rs.stringOpt("description"))
+ ).list().apply()
+ }
+
+ def update(app: App): Unit = DB localTx { implicit session =>
+ sql"""
+ update $tableName set name = ${app.name}, description = ${app.description}
+ where id = ${app.id}""".update().apply()
+ }
+
+ def delete(id: Int): Unit = DB localTx { implicit session =>
+ sql"DELETE FROM $tableName WHERE id = $id".update().apply()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
new file mode 100644
index 0000000..c9aaca5
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCChannels.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Channel
+import org.apache.predictionio.data.storage.Channels
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+/** JDBC implementation of [[Channels]] */
+class JDBCChannels(client: String, config: StorageClientConfig, prefix: String)
+ extends Channels with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "channels")
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ id serial not null primary key,
+ name text not null,
+ appid integer not null)""".execute().apply()
+ }
+
+ def insert(channel: Channel): Option[Int] = DB localTx { implicit session =>
+ val q = if (channel.id == 0) {
+ sql"INSERT INTO $tableName (name, appid) VALUES(${channel.name}, ${channel.appid})"
+ } else {
+ sql"INSERT INTO $tableName VALUES(${channel.id}, ${channel.name}, ${channel.appid})"
+ }
+ Some(q.updateAndReturnGeneratedKey().apply().toInt)
+ }
+
+ def get(id: Int): Option[Channel] = DB localTx { implicit session =>
+ sql"SELECT id, name, appid FROM $tableName WHERE id = $id".
+ map(resultToChannel).single().apply()
+ }
+
+ def getByAppid(appid: Int): Seq[Channel] = DB localTx { implicit session =>
+ sql"SELECT id, name, appid FROM $tableName WHERE appid = $appid".
+ map(resultToChannel).list().apply()
+ }
+
+ def delete(id: Int): Unit = DB localTx { implicit session =>
+ sql"DELETE FROM $tableName WHERE id = $id".update().apply()
+ }
+
+ def resultToChannel(rs: WrappedResultSet): Channel = {
+ Channel(
+ id = rs.int("id"),
+ name = rs.string("name"),
+ appid = rs.int("appid"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
new file mode 100644
index 0000000..13c374d
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEngineInstances.scala
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.EngineInstance
+import org.apache.predictionio.data.storage.EngineInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+/** JDBC implementation of [[EngineInstances]] */
+class JDBCEngineInstances(client: String, config: StorageClientConfig, prefix: String)
+ extends EngineInstances with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "engineinstances")
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ id varchar(100) not null primary key,
+ status text not null,
+ startTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ endTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ engineId text not null,
+ engineVersion text not null,
+ engineVariant text not null,
+ engineFactory text not null,
+ batch text not null,
+ env text not null,
+ sparkConf text not null,
+ datasourceParams text not null,
+ preparatorParams text not null,
+ algorithmsParams text not null,
+ servingParams text not null)""".execute().apply()
+ }
+
+ def insert(i: EngineInstance): String = DB localTx { implicit session =>
+ val id = java.util.UUID.randomUUID().toString
+ sql"""
+ INSERT INTO $tableName VALUES(
+ $id,
+ ${i.status},
+ ${i.startTime},
+ ${i.endTime},
+ ${i.engineId},
+ ${i.engineVersion},
+ ${i.engineVariant},
+ ${i.engineFactory},
+ ${i.batch},
+ ${JDBCUtils.mapToString(i.env)},
+ ${JDBCUtils.mapToString(i.sparkConf)},
+ ${i.dataSourceParams},
+ ${i.preparatorParams},
+ ${i.algorithmsParams},
+ ${i.servingParams})""".update().apply()
+ id
+ }
+
+ def get(id: String): Option[EngineInstance] = DB localTx { implicit session =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ engineId,
+ engineVersion,
+ engineVariant,
+ engineFactory,
+ batch,
+ env,
+ sparkConf,
+ datasourceParams,
+ preparatorParams,
+ algorithmsParams,
+ servingParams
+ FROM $tableName WHERE id = $id""".map(resultToEngineInstance).
+ single().apply()
+ }
+
+ def getAll(): Seq[EngineInstance] = DB localTx { implicit session =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ engineId,
+ engineVersion,
+ engineVariant,
+ engineFactory,
+ batch,
+ env,
+ sparkConf,
+ datasourceParams,
+ preparatorParams,
+ algorithmsParams,
+ servingParams
+ FROM $tableName""".map(resultToEngineInstance).list().apply()
+ }
+
+ def getLatestCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Option[EngineInstance] =
+ getCompleted(engineId, engineVersion, engineVariant).headOption
+
+ def getCompleted(
+ engineId: String,
+ engineVersion: String,
+ engineVariant: String): Seq[EngineInstance] = DB localTx { implicit s =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ engineId,
+ engineVersion,
+ engineVariant,
+ engineFactory,
+ batch,
+ env,
+ sparkConf,
+ datasourceParams,
+ preparatorParams,
+ algorithmsParams,
+ servingParams
+ FROM $tableName
+ WHERE
+ status = 'COMPLETED' AND
+ engineId = $engineId AND
+ engineVersion = $engineVersion AND
+ engineVariant = $engineVariant
+ ORDER BY startTime DESC""".
+ map(resultToEngineInstance).list().apply()
+ }
+
+ def update(i: EngineInstance): Unit = DB localTx { implicit session =>
+ sql"""
+ update $tableName set
+ status = ${i.status},
+ startTime = ${i.startTime},
+ endTime = ${i.endTime},
+ engineId = ${i.engineId},
+ engineVersion = ${i.engineVersion},
+ engineVariant = ${i.engineVariant},
+ engineFactory = ${i.engineFactory},
+ batch = ${i.batch},
+ env = ${JDBCUtils.mapToString(i.env)},
+ sparkConf = ${JDBCUtils.mapToString(i.sparkConf)},
+ datasourceParams = ${i.dataSourceParams},
+ preparatorParams = ${i.preparatorParams},
+ algorithmsParams = ${i.algorithmsParams},
+ servingParams = ${i.servingParams}
+ where id = ${i.id}""".update().apply()
+ }
+
+ def delete(id: String): Unit = DB localTx { implicit session =>
+ sql"DELETE FROM $tableName WHERE id = $id".update().apply()
+ }
+
+ /** Convert JDBC results to [[EngineInstance]] */
+ def resultToEngineInstance(rs: WrappedResultSet): EngineInstance = {
+ EngineInstance(
+ id = rs.string("id"),
+ status = rs.string("status"),
+ startTime = rs.jodaDateTime("startTime"),
+ endTime = rs.jodaDateTime("endTime"),
+ engineId = rs.string("engineId"),
+ engineVersion = rs.string("engineVersion"),
+ engineVariant = rs.string("engineVariant"),
+ engineFactory = rs.string("engineFactory"),
+ batch = rs.string("batch"),
+ env = JDBCUtils.stringToMap(rs.string("env")),
+ sparkConf = JDBCUtils.stringToMap(rs.string("sparkConf")),
+ dataSourceParams = rs.string("datasourceParams"),
+ preparatorParams = rs.string("preparatorParams"),
+ algorithmsParams = rs.string("algorithmsParams"),
+ servingParams = rs.string("servingParams"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
new file mode 100644
index 0000000..90eb5f3
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCEvaluationInstances.scala
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.EvaluationInstance
+import org.apache.predictionio.data.storage.EvaluationInstances
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+/** JDBC implementations of [[EvaluationInstances]] */
+class JDBCEvaluationInstances(client: String, config: StorageClientConfig, prefix: String)
+ extends EvaluationInstances with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "evaluationinstances")
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ id varchar(100) not null primary key,
+ status text not null,
+ startTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ endTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ evaluationClass text not null,
+ engineParamsGeneratorClass text not null,
+ batch text not null,
+ env text not null,
+ sparkConf text not null,
+ evaluatorResults text not null,
+ evaluatorResultsHTML text not null,
+ evaluatorResultsJSON text)""".execute().apply()
+ }
+
+ def insert(i: EvaluationInstance): String = DB localTx { implicit session =>
+ val id = java.util.UUID.randomUUID().toString
+ sql"""
+ INSERT INTO $tableName VALUES(
+ $id,
+ ${i.status},
+ ${i.startTime},
+ ${i.endTime},
+ ${i.evaluationClass},
+ ${i.engineParamsGeneratorClass},
+ ${i.batch},
+ ${JDBCUtils.mapToString(i.env)},
+ ${JDBCUtils.mapToString(i.sparkConf)},
+ ${i.evaluatorResults},
+ ${i.evaluatorResultsHTML},
+ ${i.evaluatorResultsJSON})""".update().apply()
+ id
+ }
+
+ def get(id: String): Option[EvaluationInstance] = DB localTx { implicit session =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ evaluationClass,
+ engineParamsGeneratorClass,
+ batch,
+ env,
+ sparkConf,
+ evaluatorResults,
+ evaluatorResultsHTML,
+ evaluatorResultsJSON
+ FROM $tableName WHERE id = $id
+ """.map(resultToEvaluationInstance).single().apply()
+ }
+
+ def getAll(): Seq[EvaluationInstance] = DB localTx { implicit session =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ evaluationClass,
+ engineParamsGeneratorClass,
+ batch,
+ env,
+ sparkConf,
+ evaluatorResults,
+ evaluatorResultsHTML,
+ evaluatorResultsJSON
+ FROM $tableName
+ """.map(resultToEvaluationInstance).list().apply()
+ }
+
+ def getCompleted(): Seq[EvaluationInstance] = DB localTx { implicit s =>
+ sql"""
+ SELECT
+ id,
+ status,
+ startTime,
+ endTime,
+ evaluationClass,
+ engineParamsGeneratorClass,
+ batch,
+ env,
+ sparkConf,
+ evaluatorResults,
+ evaluatorResultsHTML,
+ evaluatorResultsJSON
+ FROM $tableName
+ WHERE
+ status = 'EVALCOMPLETED'
+ ORDER BY starttime DESC
+ """.map(resultToEvaluationInstance).list().apply()
+ }
+
+ def update(i: EvaluationInstance): Unit = DB localTx { implicit session =>
+ sql"""
+ update $tableName set
+ status = ${i.status},
+ startTime = ${i.startTime},
+ endTime = ${i.endTime},
+ evaluationClass = ${i.evaluationClass},
+ engineParamsGeneratorClass = ${i.engineParamsGeneratorClass},
+ batch = ${i.batch},
+ env = ${JDBCUtils.mapToString(i.env)},
+ sparkConf = ${JDBCUtils.mapToString(i.sparkConf)},
+ evaluatorResults = ${i.evaluatorResults},
+ evaluatorResultsHTML = ${i.evaluatorResultsHTML},
+ evaluatorResultsJSON = ${i.evaluatorResultsJSON}
+ where id = ${i.id}""".update().apply()
+ }
+
+ def delete(id: String): Unit = DB localTx { implicit session =>
+ sql"DELETE FROM $tableName WHERE id = $id".update().apply()
+ }
+
+ /** Convert JDBC results to [[EvaluationInstance]] */
+ def resultToEvaluationInstance(rs: WrappedResultSet): EvaluationInstance = {
+ EvaluationInstance(
+ id = rs.string("id"),
+ status = rs.string("status"),
+ startTime = rs.jodaDateTime("startTime"),
+ endTime = rs.jodaDateTime("endTime"),
+ evaluationClass = rs.string("evaluationClass"),
+ engineParamsGeneratorClass = rs.string("engineParamsGeneratorClass"),
+ batch = rs.string("batch"),
+ env = JDBCUtils.stringToMap(rs.string("env")),
+ sparkConf = JDBCUtils.stringToMap(rs.string("sparkConf")),
+ evaluatorResults = rs.string("evaluatorResults"),
+ evaluatorResultsHTML = rs.string("evaluatorResultsHTML"),
+ evaluatorResultsJSON = rs.string("evaluatorResultsJSON"))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
new file mode 100644
index 0000000..dddef67
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCLEvents.scala
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.DataMap
+import org.apache.predictionio.data.storage.Event
+import org.apache.predictionio.data.storage.LEvents
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+import org.json4s.JObject
+import org.json4s.native.Serialization.read
+import org.json4s.native.Serialization.write
+import scalikejdbc._
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+/** JDBC implementation of [[LEvents]] */
+class JDBCLEvents(
+ client: String,
+ config: StorageClientConfig,
+ namespace: String) extends LEvents with Logging {
+ implicit private val formats = org.json4s.DefaultFormats
+
+ def init(appId: Int, channelId: Option[Int] = None): Boolean = {
+
+ // To use index, it must be varchar less than 255 characters on a VARCHAR column
+ val useIndex = config.properties.contains("INDEX") &&
+ config.properties("INDEX").equalsIgnoreCase("enabled")
+
+ val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
+ val entityIdIndexName = s"idx_${tableName}_ei"
+ val entityTypeIndexName = s"idx_${tableName}_et"
+ DB autoCommit { implicit session =>
+ if (useIndex) {
+ SQL(s"""
+ create table if not exists $tableName (
+ id varchar(32) not null primary key,
+ event varchar(255) not null,
+ entityType varchar(255) not null,
+ entityId varchar(255) not null,
+ targetEntityType text,
+ targetEntityId text,
+ properties text,
+ eventTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ eventTimeZone varchar(50) not null,
+ tags text,
+ prId text,
+ creationTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ creationTimeZone varchar(50) not null)""").execute().apply()
+
+ // create index
+ SQL(s"create index $entityIdIndexName on $tableName (entityId)").execute().apply()
+ SQL(s"create index $entityTypeIndexName on $tableName (entityType)").execute().apply()
+ } else {
+ SQL(s"""
+ create table if not exists $tableName (
+ id varchar(32) not null primary key,
+ event text not null,
+ entityType text not null,
+ entityId text not null,
+ targetEntityType text,
+ targetEntityId text,
+ properties text,
+ eventTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ eventTimeZone varchar(50) not null,
+ tags text,
+ prId text,
+ creationTime timestamp DEFAULT CURRENT_TIMESTAMP,
+ creationTimeZone varchar(50) not null)""").execute().apply()
+ }
+ true
+ }
+ }
+
+ def remove(appId: Int, channelId: Option[Int] = None): Boolean =
+ DB autoCommit { implicit session =>
+ SQL(s"""
+ drop table ${JDBCUtils.eventTableName(namespace, appId, channelId)}
+ """).execute().apply()
+ true
+ }
+
+ def close(): Unit = ConnectionPool.closeAll()
+
+ def futureInsert(event: Event, appId: Int, channelId: Option[Int])(
+ implicit ec: ExecutionContext): Future[String] = Future {
+ DB localTx { implicit session =>
+ val id = event.eventId.getOrElse(JDBCUtils.generateId)
+ val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
+ sql"""
+ insert into $tableName values(
+ $id,
+ ${event.event},
+ ${event.entityType},
+ ${event.entityId},
+ ${event.targetEntityType},
+ ${event.targetEntityId},
+ ${write(event.properties.toJObject)},
+ ${event.eventTime},
+ ${event.eventTime.getZone.getID},
+ ${if (event.tags.nonEmpty) Some(event.tags.mkString(",")) else None},
+ ${event.prId},
+ ${event.creationTime},
+ ${event.creationTime.getZone.getID}
+ )
+ """.update().apply()
+ id
+ }
+ }
+
+ def futureGet(eventId: String, appId: Int, channelId: Option[Int])(
+ implicit ec: ExecutionContext): Future[Option[Event]] = Future {
+ DB readOnly { implicit session =>
+ val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
+ sql"""
+ select
+ id,
+ event,
+ entityType,
+ entityId,
+ targetEntityType,
+ targetEntityId,
+ properties,
+ eventTime,
+ eventTimeZone,
+ tags,
+ prId,
+ creationTime,
+ creationTimeZone
+ from $tableName
+ where id = $eventId
+ """.map(resultToEvent).single().apply()
+ }
+ }
+
+ def futureDelete(eventId: String, appId: Int, channelId: Option[Int])(
+ implicit ec: ExecutionContext): Future[Boolean] = Future {
+ DB localTx { implicit session =>
+ val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
+ sql"""
+ delete from $tableName where id = $eventId
+ """.update().apply()
+ true
+ }
+ }
+
+ def futureFind(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None,
+ limit: Option[Int] = None,
+ reversed: Option[Boolean] = None
+ )(implicit ec: ExecutionContext): Future[Iterator[Event]] = Future {
+ DB readOnly { implicit session =>
+ val tableName = sqls.createUnsafely(JDBCUtils.eventTableName(namespace, appId, channelId))
+ val whereClause = sqls.toAndConditionOpt(
+ startTime.map(x => sqls"eventTime >= $x"),
+ untilTime.map(x => sqls"eventTime < $x"),
+ entityType.map(x => sqls"entityType = $x"),
+ entityId.map(x => sqls"entityId = $x"),
+ eventNames.map(x =>
+ sqls.toOrConditionOpt(x.map(y =>
+ Some(sqls"event = $y")
+ ): _*)
+ ).getOrElse(None),
+ targetEntityType.map(x => x.map(y => sqls"targetEntityType = $y")
+ .getOrElse(sqls"targetEntityType IS NULL")),
+ targetEntityId.map(x => x.map(y => sqls"targetEntityId = $y")
+ .getOrElse(sqls"targetEntityId IS NULL"))
+ ).map(sqls.where(_)).getOrElse(sqls"")
+ val orderByClause = reversed.map(x =>
+ if (x) sqls"eventTime desc" else sqls"eventTime asc"
+ ).getOrElse(sqls"eventTime asc")
+ val limitClause = limit.map(x =>
+ if (x < 0) sqls"" else sqls.limit(x)
+ ).getOrElse(sqls"")
+ val q = sql"""
+ select
+ id,
+ event,
+ entityType,
+ entityId,
+ targetEntityType,
+ targetEntityId,
+ properties,
+ eventTime,
+ eventTimeZone,
+ tags,
+ prId,
+ creationTime,
+ creationTimeZone
+ from $tableName
+ $whereClause
+ order by $orderByClause
+ $limitClause
+ """
+ q.map(resultToEvent).list().apply().toIterator
+ }
+ }
+
+ private[predictionio] def resultToEvent(rs: WrappedResultSet): Event = {
+ Event(
+ eventId = rs.stringOpt("id"),
+ event = rs.string("event"),
+ entityType = rs.string("entityType"),
+ entityId = rs.string("entityId"),
+ targetEntityType = rs.stringOpt("targetEntityType"),
+ targetEntityId = rs.stringOpt("targetEntityId"),
+ properties = rs.stringOpt("properties").map(p =>
+ DataMap(read[JObject](p))).getOrElse(DataMap()),
+ eventTime = new DateTime(rs.jodaDateTime("eventTime"),
+ DateTimeZone.forID(rs.string("eventTimeZone"))),
+ tags = rs.stringOpt("tags").map(t => t.split(",").toList).getOrElse(Nil),
+ prId = rs.stringOpt("prId"),
+ creationTime = new DateTime(rs.jodaDateTime("creationTime"),
+ DateTimeZone.forID(rs.string("creationTimeZone")))
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
new file mode 100644
index 0000000..b48502a
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCModels.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Model
+import org.apache.predictionio.data.storage.Models
+import org.apache.predictionio.data.storage.StorageClientConfig
+import scalikejdbc._
+
+/** JDBC implementation of [[Models]] */
+class JDBCModels(client: String, config: StorageClientConfig, prefix: String)
+ extends Models with Logging {
+ /** Database table name for this data access object */
+ val tableName = JDBCUtils.prefixTableName(prefix, "models")
+
+ /** Determines binary column type based on JDBC driver type */
+ val binaryColumnType = JDBCUtils.binaryColumnType(client)
+ DB autoCommit { implicit session =>
+ sql"""
+ create table if not exists $tableName (
+ id varchar(100) not null primary key,
+ models $binaryColumnType not null)""".execute().apply()
+ }
+
+ def insert(i: Model): Unit = DB localTx { implicit session =>
+ sql"insert into $tableName values(${i.id}, ${i.models})".update().apply()
+ }
+
+ def get(id: String): Option[Model] = DB readOnly { implicit session =>
+ sql"select id, models from $tableName where id = $id".map { r =>
+ Model(id = r.string("id"), models = r.bytes("models"))
+ }.single().apply()
+ }
+
+ def delete(id: String): Unit = DB localTx { implicit session =>
+ sql"delete from $tableName where id = $id".execute().apply()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
new file mode 100644
index 0000000..2e6ee83
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCPEvents.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import java.sql.{DriverManager, ResultSet}
+
+import com.github.nscala_time.time.Imports._
+import org.apache.predictionio.data.storage.{DataMap, Event, PEvents, StorageClientConfig}
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.{JdbcRDD, RDD}
+import org.apache.spark.sql.{SQLContext, SaveMode}
+import org.json4s.JObject
+import org.json4s.native.Serialization
+import scalikejdbc._
+
+/** JDBC implementation of [[PEvents]] */
+class JDBCPEvents(client: String, config: StorageClientConfig, namespace: String) extends PEvents {
+ @transient private implicit lazy val formats = org.json4s.DefaultFormats
+ def find(
+ appId: Int,
+ channelId: Option[Int] = None,
+ startTime: Option[DateTime] = None,
+ untilTime: Option[DateTime] = None,
+ entityType: Option[String] = None,
+ entityId: Option[String] = None,
+ eventNames: Option[Seq[String]] = None,
+ targetEntityType: Option[Option[String]] = None,
+ targetEntityId: Option[Option[String]] = None)(sc: SparkContext): RDD[Event] = {
+ val lower = startTime.map(_.getMillis).getOrElse(0.toLong)
+ /** Change the default upper bound from +100 to +1 year because MySQL's
+ * FROM_UNIXTIME(t) will return NULL if we use +100 years.
+ */
+ val upper = untilTime.map(_.getMillis).getOrElse((DateTime.now + 1.years).getMillis)
+ val par = scala.math.min(
+ new Duration(upper - lower).getStandardDays,
+ config.properties.getOrElse("PARTITIONS", "4").toLong).toInt
+ val entityTypeClause = entityType.map(x => s"and entityType = '$x'").getOrElse("")
+ val entityIdClause = entityId.map(x => s"and entityId = '$x'").getOrElse("")
+ val eventNamesClause =
+ eventNames.map("and (" + _.map(y => s"event = '$y'").mkString(" or ") + ")").getOrElse("")
+ val targetEntityTypeClause = targetEntityType.map(
+ _.map(x => s"and targetEntityType = '$x'"
+ ).getOrElse("and targetEntityType is null")).getOrElse("")
+ val targetEntityIdClause = targetEntityId.map(
+ _.map(x => s"and targetEntityId = '$x'"
+ ).getOrElse("and targetEntityId is null")).getOrElse("")
+ val q = s"""
+ select
+ id,
+ event,
+ entityType,
+ entityId,
+ targetEntityType,
+ targetEntityId,
+ properties,
+ eventTime,
+ eventTimeZone,
+ tags,
+ prId,
+ creationTime,
+ creationTimeZone
+ from ${JDBCUtils.eventTableName(namespace, appId, channelId)}
+ where
+ eventTime >= ${JDBCUtils.timestampFunction(client)}(?) and
+ eventTime < ${JDBCUtils.timestampFunction(client)}(?)
+ $entityTypeClause
+ $entityIdClause
+ $eventNamesClause
+ $targetEntityTypeClause
+ $targetEntityIdClause
+ """.replace("\n", " ")
+ new JdbcRDD(
+ sc,
+ () => {
+ DriverManager.getConnection(
+ client,
+ config.properties("USERNAME"),
+ config.properties("PASSWORD"))
+ },
+ q,
+ lower / 1000,
+ upper / 1000,
+ par,
+ (r: ResultSet) => {
+ Event(
+ eventId = Option(r.getString("id")),
+ event = r.getString("event"),
+ entityType = r.getString("entityType"),
+ entityId = r.getString("entityId"),
+ targetEntityType = Option(r.getString("targetEntityType")),
+ targetEntityId = Option(r.getString("targetEntityId")),
+ properties = Option(r.getString("properties")).map(x =>
+ DataMap(Serialization.read[JObject](x))).getOrElse(DataMap()),
+ eventTime = new DateTime(r.getTimestamp("eventTime").getTime,
+ DateTimeZone.forID(r.getString("eventTimeZone"))),
+ tags = Option(r.getString("tags")).map(x =>
+ x.split(",").toList).getOrElse(Nil),
+ prId = Option(r.getString("prId")),
+ creationTime = new DateTime(r.getTimestamp("creationTime").getTime,
+ DateTimeZone.forID(r.getString("creationTimeZone"))))
+ }).cache()
+ }
+
+ def write(events: RDD[Event], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+ val sqlContext = new SQLContext(sc)
+
+ import sqlContext.implicits._
+
+ val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
+
+ val eventTableColumns = Seq[String](
+ "id"
+ , "event"
+ , "entityType"
+ , "entityId"
+ , "targetEntityType"
+ , "targetEntityId"
+ , "properties"
+ , "eventTime"
+ , "eventTimeZone"
+ , "tags"
+ , "prId"
+ , "creationTime"
+ , "creationTimeZone")
+
+ val eventDF = events.map(x =>
+ Event(eventId = None, event = x.event, entityType = x.entityType,
+ entityId = x.entityId, targetEntityType = x.targetEntityType,
+ targetEntityId = x.targetEntityId, properties = x.properties,
+ eventTime = x.eventTime, tags = x.tags, prId= x.prId,
+ creationTime = x.eventTime)
+ )
+ .map { event =>
+ (event.eventId.getOrElse(JDBCUtils.generateId)
+ , event.event
+ , event.entityType
+ , event.entityId
+ , event.targetEntityType.orNull
+ , event.targetEntityId.orNull
+ , if (!event.properties.isEmpty) Serialization.write(event.properties.toJObject) else null
+ , new java.sql.Timestamp(event.eventTime.getMillis)
+ , event.eventTime.getZone.getID
+ , if (event.tags.nonEmpty) Some(event.tags.mkString(",")) else null
+ , event.prId
+ , new java.sql.Timestamp(event.creationTime.getMillis)
+ , event.creationTime.getZone.getID)
+ }.toDF(eventTableColumns:_*)
+
+ // spark version 1.4.0 or higher
+ val prop = new java.util.Properties
+ prop.setProperty("user", config.properties("USERNAME"))
+ prop.setProperty("password", config.properties("PASSWORD"))
+ eventDF.write.mode(SaveMode.Append).jdbc(client, tableName, prop)
+ }
+
+ def delete(eventIds: RDD[String], appId: Int, channelId: Option[Int])(sc: SparkContext): Unit = {
+
+ eventIds.foreachPartition{ iter =>
+
+ iter.foreach { eventId =>
+ DB localTx { implicit session =>
+ val tableName = JDBCUtils.eventTableName(namespace, appId, channelId)
+ val table = SQLSyntax.createUnsafely(tableName)
+ sql"""
+ delete from $table where id = $eventId
+ """.update().apply()
+ true
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
new file mode 100644
index 0000000..3eb55ba
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/JDBCUtils.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import scalikejdbc._
+
+/** JDBC related utilities */
+object JDBCUtils {
+ /** Extract JDBC driver type from URL
+ *
+ * @param url JDBC URL
+ * @return The driver type, e.g. postgresql
+ */
+ def driverType(url: String): String = {
+ val capture = """jdbc:([^:]+):""".r
+ capture findFirstIn url match {
+ case Some(capture(driverType)) => driverType
+ case None => ""
+ }
+ }
+
+ /** Determines binary column type from JDBC URL
+ *
+ * @param url JDBC URL
+ * @return Binary column type as SQLSyntax, e.g. LONGBLOB
+ */
+ def binaryColumnType(url: String): SQLSyntax = {
+ driverType(url) match {
+ case "postgresql" => sqls"bytea"
+ case "mysql" => sqls"longblob"
+ case _ => sqls"longblob"
+ }
+ }
+
+ /** Determines UNIX timestamp conversion function from JDBC URL
+ *
+ * @param url JDBC URL
+ * @return Timestamp conversion function, e.g. TO_TIMESTAMP
+ */
+ def timestampFunction(url: String): String = {
+ driverType(url) match {
+ case "postgresql" => "to_timestamp"
+ case "mysql" => "from_unixtime"
+ case _ => "from_unixtime"
+ }
+ }
+
+ /** Converts Map of String to String to comma-separated list of key=value
+ *
+ * @param m Map of String to String
+ * @return Comma-separated list, e.g. FOO=BAR,X=Y,...
+ */
+ def mapToString(m: Map[String, String]): String = {
+ m.map(t => s"${t._1}=${t._2}").mkString(",")
+ }
+
+ /** Inverse of mapToString
+ *
+ * @param str Comma-separated list, e.g. FOO=BAR,X=Y,...
+ * @return Map of String to String, e.g. Map("FOO" -> "BAR", "X" -> "Y", ...)
+ */
+ def stringToMap(str: String): Map[String, String] = {
+ str.split(",").map { x =>
+ val y = x.split("=")
+ y(0) -> y(1)
+ }.toMap[String, String]
+ }
+
+ /** Generate 32-character random ID using UUID with - stripped */
+ def generateId: String = java.util.UUID.randomUUID().toString.replace("-", "")
+
+ /** Prefix a table name
+ *
+ * @param prefix Table prefix
+ * @param table Table name
+ * @return Prefixed table name
+ */
+ def prefixTableName(prefix: String, table: String): SQLSyntax =
+ sqls.createUnsafely(s"${prefix}_$table")
+
+ /** Derive event table name
+ *
+ * @param namespace Namespace of event tables
+ * @param appId App ID
+ * @param channelId Optional channel ID
+ * @return Full event table name
+ */
+ def eventTableName(namespace: String, appId: Int, channelId: Option[Int]): String =
+ s"${namespace}_${appId}${channelId.map("_" + _).getOrElse("")}"
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
new file mode 100644
index 0000000..661e05e
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/StorageClient.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.jdbc
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.BaseStorageClient
+import org.apache.predictionio.data.storage.StorageClientConfig
+import org.apache.predictionio.data.storage.StorageClientException
+import scalikejdbc._
+
+/** JDBC implementation of [[BaseStorageClient]] */
+class StorageClient(val config: StorageClientConfig)
+ extends BaseStorageClient with Logging {
+ override val prefix = "JDBC"
+
+ if (!config.properties.contains("URL")) {
+ throw new StorageClientException("The URL variable is not set!", null)
+ }
+ if (!config.properties.contains("USERNAME")) {
+ throw new StorageClientException("The USERNAME variable is not set!", null)
+ }
+ if (!config.properties.contains("PASSWORD")) {
+ throw new StorageClientException("The PASSWORD variable is not set!", null)
+ }
+
+ // set max size of connection pool
+ val maxSize: Int = config.properties.getOrElse("CONNECTIONS", "8").toInt
+ val settings = ConnectionPoolSettings(maxSize = maxSize)
+
+ ConnectionPool.singleton(
+ config.properties("URL"),
+ config.properties("USERNAME"),
+ config.properties("PASSWORD"),
+ settings)
+ /** JDBC connection URL. Connections are managed by ScalikeJDBC. */
+ val client = config.properties("URL")
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
new file mode 100644
index 0000000..e552e54
--- /dev/null
+++ b/storage/jdbc/src/main/scala/org/apache/predictionio/data/storage/jdbc/package.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage
+
+/** JDBC implementation of storage traits, supporting meta data, event data, and
+ * model data
+ *
+ * @group Implementation
+ */
+package object jdbc {}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/jdbc/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/storage/jdbc/src/test/resources/application.conf b/storage/jdbc/src/test/resources/application.conf
new file mode 100644
index 0000000..eecae44
--- /dev/null
+++ b/storage/jdbc/src/test/resources/application.conf
@@ -0,0 +1,28 @@
+org.apache.predictionio.data.storage {
+ sources {
+ mongodb {
+ type = mongodb
+ hosts = [localhost]
+ ports = [27017]
+ }
+ elasticsearch {
+ type = elasticsearch
+ hosts = [localhost]
+ ports = [9300]
+ }
+ }
+ repositories {
+ # This section is dummy just to make storage happy.
+ # The actual testing will not bypass these repository settings completely.
+ # Please refer to StorageTestUtils.scala.
+ settings {
+ name = "test_predictionio"
+ source = mongodb
+ }
+
+ appdata {
+ name = "test_predictionio_appdata"
+ source = mongodb
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/.gitignore
----------------------------------------------------------------------
diff --git a/storage/localfs/.gitignore b/storage/localfs/.gitignore
new file mode 100644
index 0000000..ae3c172
--- /dev/null
+++ b/storage/localfs/.gitignore
@@ -0,0 +1 @@
+/bin/
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/build.sbt
----------------------------------------------------------------------
diff --git a/storage/localfs/build.sbt b/storage/localfs/build.sbt
new file mode 100644
index 0000000..2cf9977
--- /dev/null
+++ b/storage/localfs/build.sbt
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+name := "apache-predictionio-data-localfs"
+
+libraryDependencies ++= Seq(
+ "org.apache.predictionio" %% "apache-predictionio-core" % version.value % "provided",
+ "org.apache.predictionio" %% "apache-predictionio-data" % version.value % "provided",
+ "org.scalatest" %% "scalatest" % "2.1.7" % "test",
+ "org.specs2" %% "specs2" % "2.3.13" % "test")
+
+parallelExecution in Test := false
+
+pomExtra := childrenPomExtra.value
+
+assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false, includeDependency = true)
+
+assemblyMergeStrategy in assembly := {
+ case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat
+ case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+}
+
+// skip test in assembly
+test in assembly := {}
+
+outputPath in assembly := baseDirectory.value.getAbsoluteFile.getParentFile.getParentFile / "assembly" / "spark" / ("pio-data-localfs-assembly-" + version.value + ".jar")
+
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d78b3cbe/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
----------------------------------------------------------------------
diff --git a/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
new file mode 100644
index 0000000..f528af9
--- /dev/null
+++ b/storage/localfs/src/main/scala/org/apache/predictionio/data/storage/localfs/LocalFSModels.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.predictionio.data.storage.localfs
+
+import java.io.File
+import java.io.FileNotFoundException
+import java.io.FileOutputStream
+
+import grizzled.slf4j.Logging
+import org.apache.predictionio.data.storage.Model
+import org.apache.predictionio.data.storage.Models
+import org.apache.predictionio.data.storage.StorageClientConfig
+
+import scala.io.Source
+
+class LocalFSModels(f: File, config: StorageClientConfig, prefix: String)
+ extends Models with Logging {
+
+ def insert(i: Model): Unit = {
+ try {
+ val fos = new FileOutputStream(new File(f, s"${prefix}${i.id}"))
+ fos.write(i.models)
+ fos.close
+ } catch {
+ case e: FileNotFoundException => error(e.getMessage)
+ }
+ }
+
+ def get(id: String): Option[Model] = {
+ try {
+ Some(Model(
+ id = id,
+ models = Source.fromFile(new File(f, s"${prefix}${id}"))(
+ scala.io.Codec.ISO8859).map(_.toByte).toArray))
+ } catch {
+ case e: Throwable =>
+ error(e.getMessage)
+ None
+ }
+ }
+
+ def delete(id: String): Unit = {
+ val m = new File(f, s"${prefix}${id}")
+ if (!m.delete) error(s"Unable to delete ${m.getCanonicalPath}!")
+ }
+}
[32/50] [abbrv] incubator-predictionio git commit: Print stack trace
to pio.log
Posted by do...@apache.org.
Print stack trace to pio.log
Closes #360
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/a4edda1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/a4edda1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/a4edda1f
Branch: refs/heads/livedoc
Commit: a4edda1fa04c1a81fc57e20cfd312c0314626eea
Parents: 00779c3
Author: Shinsuke Sugaya <sh...@yahoo.co.jp>
Authored: Mon Mar 20 11:01:09 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Mon Mar 20 11:01:09 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/predictionio/tools/commands/Management.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/a4edda1f/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
index 10aca41..47c9f3b 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/commands/Management.scala
@@ -170,7 +170,7 @@ object Management extends EitherLogging {
} getOrElse {
"No properly configured storage backend sources."
}
- return logAndFail(errStr + sources)
+ return logOnFail(errStr + sources, e)
}
info("Your system is all ready to go.")
Right(pioStatus)
[20/50] [abbrv] incubator-predictionio git commit: Show running
containers after tests has finished
Posted by do...@apache.org.
Show running containers after tests has finished
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/b6709dbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/b6709dbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/b6709dbc
Branch: refs/heads/livedoc
Commit: b6709dbc83d71600270337a70b3a13ca2cde339a
Parents: 2da30b0
Author: Donald Szeto <do...@apache.org>
Authored: Sun Mar 12 17:04:26 2017 -0700
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Mar 12 17:04:26 2017 -0700
----------------------------------------------------------------------
tests/after_script.travis.sh | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/b6709dbc/tests/after_script.travis.sh
----------------------------------------------------------------------
diff --git a/tests/after_script.travis.sh b/tests/after_script.travis.sh
index fad9bf4..005d410 100755
--- a/tests/after_script.travis.sh
+++ b/tests/after_script.travis.sh
@@ -16,4 +16,8 @@
# limitations under the License.
#
+# Print a summary of containers used
+docker ps -a
+
+# Clean up used containers
docker-compose -f tests/docker-compose.yml down
[04/50] [abbrv] incubator-predictionio git commit: Unit is not Unit
value
Posted by do...@apache.org.
Unit is not Unit value
Closes #347
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/4e3d1993
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/4e3d1993
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/4e3d1993
Branch: refs/heads/livedoc
Commit: 4e3d1993e5b8a401c88192f7ef0a23bd7063c0f2
Parents: cc5bb91
Author: Naoki Takezoe <ta...@gmail.com>
Authored: Sun Mar 5 22:42:17 2017 -0800
Committer: Donald Szeto <do...@apache.org>
Committed: Sun Mar 5 22:42:17 2017 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/predictionio/controller/Engine.scala | 4 ++--
.../scala/org/apache/predictionio/controller/LAlgorithm.scala | 2 +-
.../org/apache/predictionio/controller/P2LAlgorithm.scala | 2 +-
.../scala/org/apache/predictionio/controller/PAlgorithm.scala | 4 ++--
.../scala/org/apache/predictionio/core/BaseAlgorithm.scala | 2 +-
.../scala/org/apache/predictionio/workflow/CreateServer.scala | 2 +-
.../scala/org/apache/predictionio/workflow/FakeWorkflow.scala | 2 +-
.../scala/org/apache/predictionio/controller/EngineTest.scala | 6 +++---
.../src/main/scala/org/apache/predictionio/tools/Runner.scala | 4 ++--
9 files changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
index 881273c..436c542 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/Engine.scala
@@ -208,7 +208,7 @@ class Engine[TD, EI, PD, Q, P, A](
Doer(algorithmClassMap(algoName), algoParams)
}
- val models = if (persistedModels.exists(m => m.isInstanceOf[Unit.type])) {
+ val models = if (persistedModels.exists(m => m.isInstanceOf[Unit])) {
// If any of persistedModels is Unit, we need to re-train the model.
logger.info("Some persisted models are Unit, need to re-train.")
val (dataSourceName, dataSourceParams) = engineParams.dataSourceParams
@@ -222,7 +222,7 @@ class Engine[TD, EI, PD, Q, P, A](
val models = algorithms.zip(persistedModels).map { case (algo, m) =>
m match {
- case Unit => algo.trainBase(sc, pd)
+ case () => algo.trainBase(sc, pd)
case _ => m
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
index 32b7d0c..9e973e4 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/LAlgorithm.scala
@@ -124,7 +124,7 @@ abstract class LAlgorithm[PD, M : ClassTag, Q, P]
modelId, algoParams, sc)) {
PersistentModelManifest(className = m.getClass.getName)
} else {
- Unit
+ ()
}
} else {
m
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
index 3a9e61c..ede8dc2 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/P2LAlgorithm.scala
@@ -115,7 +115,7 @@ abstract class P2LAlgorithm[PD, M: ClassTag, Q: ClassTag, P]
modelId, algoParams, sc)) {
PersistentModelManifest(className = m.getClass.getName)
} else {
- Unit
+ ()
}
} else {
m
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
index 4a3e012..3419de3 100644
--- a/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/controller/PAlgorithm.scala
@@ -120,10 +120,10 @@ abstract class PAlgorithm[PD, M, Q, P]
modelId, algoParams, sc)) {
PersistentModelManifest(className = m.getClass.getName)
} else {
- Unit
+ ()
}
} else {
- Unit
+ ()
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/core/BaseAlgorithm.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/core/BaseAlgorithm.scala b/core/src/main/scala/org/apache/predictionio/core/BaseAlgorithm.scala
index 0c9faa7..8b9edc1 100644
--- a/core/src/main/scala/org/apache/predictionio/core/BaseAlgorithm.scala
+++ b/core/src/main/scala/org/apache/predictionio/core/BaseAlgorithm.scala
@@ -112,7 +112,7 @@ abstract class BaseAlgorithm[PD, M, Q, P]
sc: SparkContext,
modelId: String,
algoParams: Params,
- bm: Any): Any = Unit
+ bm: Any): Any = ()
/** :: DeveloperApi ::
* Obtains the type signature of query for this algorithm
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
index ddc8d90..6f274bc 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/CreateServer.scala
@@ -268,7 +268,7 @@ class MasterActor (
.param(ServerKey.param, ServerKey.get)
.method("POST").asString.code
code match {
- case 200 => Unit
+ case 200 => ()
case 404 => log.error(
s"Another process is using $serverUrl. Unable to undeploy.")
case _ => log.error(
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala b/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
index 69a5f59..c9b1d23 100644
--- a/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
+++ b/core/src/main/scala/org/apache/predictionio/workflow/FakeWorkflow.scala
@@ -102,7 +102,7 @@ trait FakeRun extends Evaluation with EngineParamsGenerator {
engineParamsList = Seq(new EngineParams())
}
- def func: (SparkContext => Unit) = { (sc: SparkContext) => Unit }
+ def func: (SparkContext => Unit) = { (sc: SparkContext) => () }
def func_=(f: SparkContext => Unit) {
runner = new FakeRunner(f)
}
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala b/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
index 65a363c..94879a5 100644
--- a/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
+++ b/core/src/test/scala/org/apache/predictionio/controller/EngineTest.scala
@@ -60,7 +60,7 @@ extends FunSuite with Inside with SharedSparkContext {
// PAlgo2.Model doesn't have IPersistentModel trait implemented. Hence the
// model extract after train is Unit.
- models should contain theSameElementsAs Seq(Unit)
+ models should contain theSameElementsAs Seq(())
}
test("Engine.train persisting PAlgo.Model") {
@@ -96,7 +96,7 @@ extends FunSuite with Inside with SharedSparkContext {
val pModel21 = PersistentModelManifest(model21.getClass.getName)
val pModel22 = PersistentModelManifest(model22.getClass.getName)
- models should contain theSameElementsAs Seq(Unit, pModel21, pModel22)
+ models should contain theSameElementsAs Seq((), pModel21, pModel22)
}
test("Engine.train persisting LAlgo.Model") {
@@ -181,7 +181,7 @@ extends FunSuite with Inside with SharedSparkContext {
val pModel23 = PersistentModelManifest(model23.getClass.getName)
models should contain theSameElementsAs Seq(
- Unit, pModel21, pModel22, pModel23, model24, model25)
+ (), pModel21, pModel22, pModel23, model24, model25)
}
test("Engine.eval") {
http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4e3d1993/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
index f98dda1..4f5a176 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Runner.scala
@@ -73,7 +73,7 @@ object Runner extends EitherLogging {
(fs, uri) match {
case (Some(f), Some(u)) =>
f.close()
- case _ => Unit
+ case _ => ()
}
}
@@ -114,7 +114,7 @@ object Runner extends EitherLogging {
case (_, "cluster", m) if m.startsWith("spark://") =>
return logAndFail(
"Using cluster deploy mode with Spark standalone cluster is not supported")
- case _ => Unit
+ case _ => ()
}
// Initialize HDFS API for scratch URI